diff --git a/.github/actions/cache/package-lock.json b/.github/actions/cache/package-lock.json index d452b53dccbc22..3c3887ba4b29c3 100644 --- a/.github/actions/cache/package-lock.json +++ b/.github/actions/cache/package-lock.json @@ -4283,9 +4283,9 @@ } }, "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dev": true, "license": "MIT", "dependencies": { diff --git a/docs/articles_en/about-openvino/performance-benchmarks.rst b/docs/articles_en/about-openvino/performance-benchmarks.rst index 8a58dc27df1f83..5d9abfe891584f 100644 --- a/docs/articles_en/about-openvino/performance-benchmarks.rst +++ b/docs/articles_en/about-openvino/performance-benchmarks.rst @@ -64,7 +64,7 @@ implemented in your solutions. Click the buttons below to see the chosen benchma :outline: :expand: - :material-regular:`bar_chart;1.4em` OVMS for GenAI (coming soon) + :material-regular:`bar_chart;1.4em` OVMS for GenAI @@ -163,7 +163,7 @@ For a listing of all platforms and configurations used for testing, refer to the 2024.5, as of November 20, 2024. * OpenVINO Model Server performance results are based on release - 2024.4, as of Sept. 30, 2024. + 2024.5, as of November 20, 2024. The results may not reflect all publicly available updates. Intel technologies' features and benefits depend on system configuration and may require enabled hardware, software, or service diff --git a/docs/articles_en/about-openvino/release-notes-openvino.rst b/docs/articles_en/about-openvino/release-notes-openvino.rst index 343c9e780f05dc..9e7673d7d0910d 100644 --- a/docs/articles_en/about-openvino/release-notes-openvino.rst +++ b/docs/articles_en/about-openvino/release-notes-openvino.rst @@ -32,7 +32,7 @@ What's new * New models supported: Llama 3.2 (1B & 3B), Gemma 2 (2B & 9B), and YOLO11. * LLM support on NPU: Llama 3 8B, Llama 2 7B, Mistral-v0.2-7B, Qwen2-7B-Instruct and Phi-3 - Mini-Instruct. + Mini-Instruct. * Noteworthy notebooks added: Sam2, Llama3.2, Llama3.2 - Vision, Wav2Lip, Whisper, and Llava. * Preview: support for Flax, a high-performance Python neural network library based on JAX. Its modular design allows for easy customization and accelerated inference on GPUs. @@ -87,8 +87,8 @@ Common * A new constant constructor has been added, enabling constants to be created from data pointer as shared memory. Additionally, it can take ownership of a shared, or other, object, avoiding a two-step process to wrap memory into ``ov::Tensor``. -* Files are now read via the async ReadFile API, reducing the bottleneck for LLM model load - times on GPU. +* Asynchronous file reading with mmap library has been implemented, reducing loading times for + model files, especially for LLMs. * CPU implementation of SliceScatter operator is now available, used for models such as Gemma, supporting increased LLM performance. diff --git a/docs/articles_en/about-openvino/release-notes-openvino/system-requirements.rst b/docs/articles_en/about-openvino/release-notes-openvino/system-requirements.rst index a12cacf8402953..79a9f63821c16f 100644 --- a/docs/articles_en/about-openvino/release-notes-openvino/system-requirements.rst +++ b/docs/articles_en/about-openvino/release-notes-openvino/system-requirements.rst @@ -37,7 +37,7 @@ CPU * Ubuntu 20.04 long-term support (LTS), 64-bit (Kernel 5.15+) * macOS 12.6 and above, 64-bit and ARM64 * CentOS 7 - * Red Hat Enterprise Linux 9.3-9.4, 64-bit + * Red Hat Enterprise Linux (RHEL) 8 and 9, 64-bit * openSUSE Tumbleweed, 64-bit and ARM64 * Ubuntu 20.04 ARM64 @@ -65,7 +65,7 @@ GPU * Ubuntu 22.04 long-term support (LTS), 64-bit * Ubuntu 20.04 long-term support (LTS), 64-bit * CentOS 7 - * Red Hat Enterprise Linux 9.3-9.4, 64-bit + * Red Hat Enterprise Linux (RHEL) 8 and 9, 64-bit .. tab-item:: Additional considerations diff --git a/docs/articles_en/get-started/install-openvino.rst b/docs/articles_en/get-started/install-openvino.rst index be00804faa01d2..48ea0a434c5388 100644 --- a/docs/articles_en/get-started/install-openvino.rst +++ b/docs/articles_en/get-started/install-openvino.rst @@ -21,7 +21,7 @@ Install OpenVINO™ 2024.5 - + OpenVINO 2024.5, described here, is not a Long-Term-Support version! All currently supported versions are: diff --git a/docs/articles_en/learn-openvino.rst b/docs/articles_en/learn-openvino.rst index 4fca64051003a7..98797c9c67c126 100644 --- a/docs/articles_en/learn-openvino.rst +++ b/docs/articles_en/learn-openvino.rst @@ -14,7 +14,7 @@ Learn OpenVINO Interactive Tutorials (Python) Sample Applications (Python & C++) - Large Language Model Inference Guide + Generative AI workflow @@ -29,5 +29,5 @@ as well as an experienced user. | :doc:`OpenVINO Samples ` | The OpenVINO samples (Python and C++) are simple console applications that show how to use specific OpenVINO API features. They can assist you in executing tasks such as loading a model, running inference, querying particular device capabilities, etc. -| :doc:`Large Language Models in OpenVINO ` +| :doc:`Generative AI workflow ` | Detailed information on how OpenVINO accelerates Generative AI use cases and what models it supports. This tutorial provides instructions for running Generative AI models using Hugging Face Optimum Intel and Native OpenVINO APIs. diff --git a/docs/articles_en/learn-openvino/llm_inference_guide.rst b/docs/articles_en/learn-openvino/llm_inference_guide.rst index 36c001c015f744..bfc4f9b4c49173 100644 --- a/docs/articles_en/learn-openvino/llm_inference_guide.rst +++ b/docs/articles_en/learn-openvino/llm_inference_guide.rst @@ -1,140 +1,106 @@ -Large Language Model Inference Guide +Generative AI workflow ======================================== .. meta:: - :description: Explore learning materials, including interactive - Python tutorials and sample console applications that explain - how to use OpenVINO features. + :description: learn how to use OpenVINO to run generative AI models. .. toctree:: :maxdepth: 1 :hidden: - Run LLMs with Optimum Intel - Run LLMs on OpenVINO GenAI Flavor - Run LLMs on Base OpenVINO + Inference with OpenVINO GenAI + Inference with Optimum Intel + Generative AI with Base OpenVINO (not recommended) OpenVINO Tokenizers -Large Language Models (LLMs) like GPT are transformative deep learning networks capable of a -broad range of natural language tasks, from text generation to language translation. OpenVINO -optimizes the deployment of these models, enhancing their performance and integration into -various applications. This guide shows how to use LLMs with OpenVINO, from model loading and -conversion to advanced use cases. + + +Generative AI is a specific area of Deep Learning models used for producing new and “original” +data, based on input in the form of image, sound, or natural language text. Due to their +complexity and size, generative AI pipelines are more difficult to deploy and run efficiently. +OpenVINO simplifies the process and ensures high-performance integrations, with the following +options: + +.. tab-set:: + + .. tab-item:: OpenVINO GenAI + + | - Suggested for production deployment for the supported use cases. + | - Smaller footprint and fewer dependencies. + | - More optimization and customization options. + | - Available in both Python and C++. + | - A limited set of supported use cases. + + :doc:`Install the OpenVINO GenAI package <../get-started/install-openvino/install-openvino-genai>` + and run generative models out of the box. With custom + API and tokenizers, among other components, it manages the essential tasks such as the + text generation loop, tokenization, and scheduling, offering ease of use and high + performance. + + .. tab-item:: Hugging Face integration + + | - Suggested for prototyping and, if the use case is not covered by OpenVINO GenAI, production. + | - Bigger footprint and more dependencies. + | - Limited customization due to Hugging Face dependency. + | - Not usable for C++ applications. + | - A very wide range of supported models. + + Using Optimum Intel is a great way to experiment with different models and scenarios, + thanks to a simple interface for the popular API and infrastructure offered by Hugging Face. + It also enables weight compression with + `Neural Network Compression Framework (NNCF) `__, + as well as conversion on the fly. For integration with the final product it may offer + lower performance, though. + +`Check out the GenAI Quick-start Guide [PDF] `__ The advantages of using OpenVINO for LLM deployment: -* **OpenVINO offers optimized LLM inference**: - provides a full C/C++ API, leading to faster operation than Python-based runtimes; includes a - Python API for rapid development, with the option for further optimization in C++. -* **Compatible with diverse hardware**: - supports CPUs, GPUs, and neural accelerators across ARM and x86/x64 architectures, integrated - Intel® Processor Graphics, discrete Intel® Arc™ A-Series Graphics, and discrete Intel® Data - Center GPU Flex Series; features automated optimization to maximize performance on target - hardware. -* **Requires fewer dependencies**: - than frameworks like Hugging Face and PyTorch, resulting in a smaller binary size and reduced - memory footprint, making deployments easier and updates more manageable. -* **Provides compression and precision management techniques**: - such as 8-bit and 4-bit weight compression, including embedding layers, and storage format - reduction. This includes fp16 precision for non-compressed models and int8/int4 for compressed - models, like GPTQ models from `Hugging Face `__. -* **Supports a wide range of deep learning models and architectures**: - including text, image, and audio generative models like Llama 2, MPT, OPT, Stable Diffusion, - Stable Diffusion XL. This enables the development of multimodal applications, allowing for - write-once, deploy-anywhere capabilities. -* **Enhances inference capabilities**: - fused inference primitives such as Scaled Dot Product Attention, Rotary Positional Embedding, - Group Query Attention, and Mixture of Experts. It also offers advanced features like in-place - KV-cache, dynamic quantization, KV-cache quantization and encapsulation, dynamic beam size - configuration, and speculative sampling. -* **Provides stateful model optimization**: - models from the Hugging Face Transformers are converted into a stateful form, optimizing - inference performance and memory usage in long-running text generation tasks by managing past - KV-cache tensors more efficiently internally. This feature is automatically activated for many - supported models, while unsupported ones remain stateless. Learn more about the - :doc:`Stateful models and State API <../openvino-workflow/running-inference/stateful-models>`. - -OpenVINO offers three main paths for Generative AI use cases: - -* **Hugging Face**: use OpenVINO as a backend for Hugging Face frameworks (transformers, - diffusers) through the `Optimum Intel `__ - extension. -* **OpenVINO GenAI Flavor**: use OpenVINO GenAI APIs (Python and C++). -* **Base OpenVINO**: use OpenVINO native APIs (Python and C++) with - `custom pipeline code `__. - -In both cases, the OpenVINO runtime is used for inference, and OpenVINO tools are used for -optimization. The main differences are in footprint size, ease of use, and customizability. - -The Hugging Face API is easy to learn, provides a simple interface and hides the complexity of -model initialization and text generation for a better developer experience. However, it has more -dependencies, less customization, and cannot be ported to C/C++. - -The OpenVINO GenAI Flavor reduces the complexity of LLMs implementation by -automatically managing essential tasks like the text generation loop, tokenization, -and scheduling. The Native OpenVINO API provides a more hands-on experience, -requiring manual setup of these functions. Both methods are designed to minimize dependencies -and the overall application footprint and enable the use of generative models in C++ applications. - -It is recommended to start with Hugging Face frameworks to experiment with different models and -scenarios. Then the model can be used with OpenVINO APIs if it needs to be optimized -further. Optimum Intel provides interfaces that enable model optimization (weight compression) -using `Neural Network Compression Framework (NNCF) `__, -and export models to the OpenVINO model format for use in native API applications. - -Proceed to run LLMs with: +.. dropdown:: Fewer dependencies and smaller footprint + :animate: fade-in-slide-down + :color: secondary + + Less bloated than frameworks such as Hugging Face and PyTorch, with a smaller binary size and reduced + memory footprint, makes deployments easier and updates more manageable. + +.. dropdown:: Compression and precision management + :animate: fade-in-slide-down + :color: secondary + + Techniques such as 8-bit and 4-bit weight compression, including embedding layers, and storage + format reduction. This includes fp16 precision for non-compressed models and int8/int4 for + compressed models, like GPTQ models from `Hugging Face `__. + +.. dropdown:: Enhanced inference capabilities + :animate: fade-in-slide-down + :color: secondary + + Advanced features like in-place KV-cache, dynamic quantization, KV-cache quantization and + encapsulation, dynamic beam size configuration, and speculative sampling, and more are + available. + +.. dropdown:: Stateful model optimization + :animate: fade-in-slide-down + :color: secondary + + Models from the Hugging Face Transformers are converted into a stateful form, optimizing + inference performance and memory usage in long-running text generation tasks by managing past + KV-cache tensors more efficiently internally. This feature is automatically activated for + many supported models, while unsupported ones remain stateless. Learn more about the + :doc:`Stateful models and State API <../openvino-workflow/running-inference/stateful-models>`. + +.. dropdown:: Optimized LLM inference + :animate: fade-in-slide-down + :color: secondary + + Includes a Python API for rapid development and C++ for further optimization, offering + better performance than Python-based runtimes. + + +Proceed to guides on: -* :doc:`Hugging Face and Optimum Intel <./llm_inference_guide/llm-inference-hf>` * :doc:`OpenVINO GenAI Flavor <./llm_inference_guide/genai-guide>` -* :doc:`Native OpenVINO API <./llm_inference_guide/llm-inference-native-ov>` - -The table below summarizes the differences between Hugging Face and the native OpenVINO API -approaches. - -.. dropdown:: Differences between Hugging Face and the native OpenVINO API - - .. list-table:: - :widths: 20 25 55 - :header-rows: 1 - - * - - - Hugging Face through OpenVINO - - OpenVINO Native API - * - Model support - - Supports transformer-based models such as LLMs - - Supports all model architectures from most frameworks - * - APIs - - Python (Hugging Face API) - - Python, C++ (OpenVINO API) - * - Model Format - - Source Framework / OpenVINO - - Source Framework / OpenVINO - * - Inference code - - Hugging Face based - - Custom inference pipelines - * - Additional dependencies - - Many Hugging Face dependencies - - Lightweight (e.g. numpy, etc.) - * - Application footprint - - Large - - Small - * - Pre/post-processing and glue code - - Provided through high-level Hugging Face APIs - - Must be custom implemented (see OpenVINO samples and notebooks) - * - Performance - - Good, but less efficient compared to native APIs - - Inherent speed advantage with C++, but requires hands-on optimization - * - Flexibility - - Constrained to Hugging Face API - - High flexibility with Python and C++; allows custom coding - * - Learning Curve and Effort - - Lower learning curve; quick to integrate - - Higher learning curve; requires more effort in integration - * - Ideal Use Case - - Ideal for quick prototyping and Python-centric projects - - Best suited for high-performance, resource-optimized production environments - * - Model Serving - - Paid service, based on CPU/GPU usage with Hugging Face - - Free code solution, run script for own server; costs may incur for cloud services - like AWS but generally cheaper than Hugging Face rates +* :doc:`Hugging Face and Optimum Intel <./llm_inference_guide/llm-inference-hf>` + + diff --git a/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide-npu.rst b/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide-npu.rst index 5a641300a68edb..d725b306d57908 100644 --- a/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide-npu.rst +++ b/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide-npu.rst @@ -1,4 +1,4 @@ -Run LLMs with OpenVINO GenAI Flavor on NPU +Inference with OpenVINO GenAI ========================================== .. meta:: @@ -20,21 +20,22 @@ Install required dependencies: pip install nncf==2.12 onnx==1.16.1 optimum-intel==1.19.0 pip install --pre openvino openvino-tokenizers openvino-genai --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly -NOTE that for systems based on Intel® Core Ultra Processors Series 2 and 16 GB of RAM, -prompts longer then 1024 characters will not work with a model of 7B or more parameters, +Note that for systems based on Intel® Core™ Ultra Processors Series 2, more than 16GB of RAM +may be required to run prompts over 1024 tokens on models exceeding 7B parameters, such as Llama-2-7B, Mistral-0.2-7B, and Qwen-2-7B. Export an LLM model via Hugging Face Optimum-Intel ################################################## -Since **symmetrically-quantized 4-bit (INT4) models are preffered for inference on NPU**, make sure to export -the model with the proper conversion and optimization settings. +Since **symmetrically-quantized 4-bit (INT4) models are preffered for inference on NPU**, make +sure to export the model with the proper conversion and optimization settings. | You may export LLMs via Optimum-Intel, using one of two compression methods: | **group quantization** - for both smaller and larger models, | **channel-wise quantization** - remarkably effective but for models exceeding 1 billion parameters. -You select one of the methods by setting the ``--group-size`` parameter to either ``128`` or ``-1``, respectively. See the following examples: +You select one of the methods by setting the ``--group-size`` parameter to either ``128`` or +``-1``, respectively. See the following examples: .. tab-set:: diff --git a/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide.rst b/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide.rst index ebd4667d544616..9998b3989486d2 100644 --- a/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide.rst +++ b/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide.rst @@ -1,4 +1,4 @@ -Run LLM Inference on OpenVINO with the GenAI Flavor +Inference with OpenVINO GenAI =============================================================================================== .. meta:: @@ -9,39 +9,326 @@ Run LLM Inference on OpenVINO with the GenAI Flavor :hidden: NPU inference of LLMs - genai-guide/genai-use-cases -This guide will show you how to integrate the OpenVINO GenAI flavor into your application, covering -loading a model and passing the input context to receive generated text. Note that the vanilla flavor of OpenVINO -will not work with these instructions, make sure to -:doc:`install OpenVINO GenAI <../../get-started/install-openvino/install-openvino-genai>`. +This article provides reference code and guidance on running generative AI models, +using OpenVINO GenAI. Note that the base OpenVINO version will not work with these instructions, +make sure to :doc:`install OpenVINO GenAI <../../get-started/install-openvino/install-openvino-genai>`. -.. note:: +| Here is sample code for several Generative AI use case scenarios. Note that these are very basic + examples and may need adjustments for your specific needs, like changing the inference device. +| For a more extensive instruction and additional options, see the + `step-by-step chat-bot guide <#chat-bot-use-case-step-by-step>`__ below. - The examples use the CPU as the target device, however, the GPU is also supported. - Note that for the LLM pipeline, the GPU is used only for inference, while token selection, tokenization, and - detokenization remain on the CPU, for efficiency. Tokenizers are represented as a separate model and also run - on the CPU. +.. dropdown:: Text-to-Image Generation -1. Export an LLM model via Hugging Face Optimum-Intel. A chat-tuned TinyLlama model is used in this example: + .. tab-set:: + + .. tab-item:: Python + :sync: python + + .. tab-set:: + + .. tab-item:: main.py + :name: mainpy + + .. code-block:: python + + import openvino_genai + from PIL import Image + import numpy as np + + class Generator(openvino_genai.Generator): + def __init__(self, seed, mu=0.0, sigma=1.0): + openvino_genai.Generator.__init__(self) + np.random.seed(seed) + self.mu = mu + self.sigma = sigma + + def next(self): + return np.random.normal(self.mu, self.sigma) + + + def infer(model_dir: str, prompt: str): + device = 'CPU' # GPU can be used as well + random_generator = Generator(42) + pipe = openvino_genai.Text2ImagePipeline(model_dir, device) + image_tensor = pipe.generate( + prompt, + width=512, + height=512, + num_inference_steps=20, + num_images_per_prompt=1, + random_generator=random_generator + ) + + image = Image.fromarray(image_tensor.data[0]) + image.save("image.bmp") + + .. tab-item:: LoRA.py + :name: lorapy + + .. code-block:: python + + import openvino as ov + import openvino_genai + import numpy as np + import sys + + + class Generator(openvino_genai.Generator): + def __init__(self, seed, mu=0.0, sigma=1.0): + openvino_genai.Generator.__init__(self) + np.random.seed(seed) + self.mu = mu + self.sigma = sigma + + def next(self): + return np.random.normal(self.mu, self.sigma) + + + def image_write(path: str, image_tensor: ov.Tensor): + from PIL import Image + image = Image.fromarray(image_tensor.data[0]) + image.save(path) + + + def infer(models_path: str, prompt: str): + prompt = "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" + + device = "CPU" # GPU, NPU can be used as well + adapter_config = openvino_genai.AdapterConfig() + + for i in range(int(len(adapters) / 2)): + adapter = openvino_genai.Adapter(adapters[2 * i]) + alpha = float(adapters[2 * i + 1]) + adapter_config.add(adapter, alpha) + + pipe = openvino_genai.Text2ImagePipeline(models_path, device, adapters=adapter_config) + print("Generating image with LoRA adapters applied, resulting image will be in lora.bmp") + image = pipe.generate(prompt, + random_generator=Generator(42), + width=512, + height=896, + num_inference_steps=20) + + image_write("lora.bmp", image) + print("Generating image without LoRA adapters applied, resulting image will be in baseline.bmp") + image = pipe.generate(prompt, + adapters=openvino_genai.AdapterConfig(), + random_generator=Generator(42), + width=512, + height=896, + num_inference_steps=20 + ) + image_write("baseline.bmp", image) + + For more information, refer to the + `Python sample `__ + + .. tab-item:: C++ + :sync: cpp + + .. tab-set:: + + .. tab-item:: main.cpp + :name: maincpp + + .. code-block:: cpp + + #include "openvino/genai/text2image/pipeline.hpp" + + #include "imwrite.hpp" + + int32_t main(int32_t argc, char* argv[]) try { + OPENVINO_ASSERT(argc == 3, "Usage: ", argv[0], " ''"); + + const std::string models_path = argv[1], prompt = argv[2]; + const std::string device = "CPU"; // GPU, NPU can be used as well + + ov::genai::Text2ImagePipeline pipe(models_path, device); + ov::Tensor image = pipe.generate(prompt, + ov::genai::width(512), + ov::genai::height(512), + ov::genai::num_inference_steps(20), + ov::genai::num_images_per_prompt(1)); + + imwrite("image_%d.bmp", image, true); + + return EXIT_SUCCESS; + } catch (const std::exception& error) { + try { + std::cerr << error.what() << '\n'; + } catch (const std::ios_base::failure&) {} + return EXIT_FAILURE; + } catch (...) { + try { + std::cerr << "Non-exception object thrown\n"; + } catch (const std::ios_base::failure&) {} + return EXIT_FAILURE; + } + + .. tab-item:: LoRA.cpp + :name: loracpp + + .. code-block:: cpp + + #include "openvino/genai/text2image/pipeline.hpp" + + #include "imwrite.hpp" + + int32_t main(int32_t argc, char* argv[]) try { + OPENVINO_ASSERT(argc >= 3 && (argc - 3) % 2 == 0, "Usage: ", argv[0], " '' [ ...]]"); + + const std::string models_path = argv[1], prompt = argv[2]; + const std::string device = "CPU"; // GPU, NPU can be used as well + + ov::genai::AdapterConfig adapter_config; + for(size_t i = 0; i < (argc - 3)/2; ++i) { + ov::genai::Adapter adapter(argv[3 + 2*i]); + float alpha = std::atof(argv[3 + 2*i + 1]); + adapter_config.add(adapter, alpha); + } + + ov::genai::Text2ImagePipeline pipe(models_path, device, ov::genai::adapters(adapter_config)); + + std::cout << "Generating image with LoRA adapters applied, resulting image will be in lora.bmp\n"; + ov::Tensor image = pipe.generate(prompt, + ov::genai::random_generator(std::make_shared(42)), + ov::genai::width(512), + ov::genai::height(896), + ov::genai::num_inference_steps(20)); + imwrite("lora.bmp", image, true); + + std::cout << "Generating image without LoRA adapters applied, resulting image will be in baseline.bmp\n"; + image = pipe.generate(prompt, + ov::genai::adapters(), + ov::genai::random_generator(std::make_shared(42)), + ov::genai::width(512), + ov::genai::height(896), + ov::genai::num_inference_steps(20)); + imwrite("baseline.bmp", image, true); + + return EXIT_SUCCESS; + } catch (const std::exception& error) { + try { + std::cerr << error.what() << '\n'; + } catch (const std::ios_base::failure&) {} + return EXIT_FAILURE; + } catch (...) { + try { + std::cerr << "Non-exception object thrown\n"; + } catch (const std::ios_base::failure&) {} + return EXIT_FAILURE; + } + + For more information, refer to the + `C++ sample `__ + + +.. dropdown:: Speech Recognition - .. code-block:: python + The application performs inference on speech recognition Whisper Models. The samples include + the ``WhisperPipeline`` class and use audio files in WAV format at a sampling rate of 16 kHz + as input. - optimum-cli export openvino --model "TinyLlama/TinyLlama-1.1B-Chat-v1.0" --weight-format fp16 --trust-remote-code "TinyLlama-1.1B-Chat-v1.0" + .. tab-set:: + + .. tab-item:: Python + :sync: cpp + + .. code-block:: python + + import openvino_genai + import librosa - *Optional*. Optimize the model: - The model is an optimized OpenVINO IR with FP16 precision. For enhanced LLM performance, - it is recommended to use lower precision for model weights, such as INT4, and to compress weights - using NNCF during model export directly: + def read_wav(filepath): + raw_speech, samplerate = librosa.load(filepath, sr=16000) + return raw_speech.tolist() - .. code-block:: python - optimum-cli export openvino --model "TinyLlama/TinyLlama-1.1B-Chat-v1.0" --weight-format int4 --trust-remote-code "TinyLlama-1.1B-Chat-v1.0" + def infer(model_dir: str, wav_file_path: str): + device = "CPU" # GPU or NPU can be used as well. + pipe = openvino_genai.WhisperPipeline(model_dir, device) + # The pipeline expects normalized audio with a sampling rate of 16kHz. + raw_speech = read_wav(wav_file_path) + result = pipe.generate( + raw_speech, + max_new_tokens=100, + language="<|en|>", + task="transcribe", + return_timestamps=True, + ) -2. Perform generation using the new GenAI API: + print(result) + + for chunk in result.chunks: + print(f"timestamps: [{chunk.start_ts}, {chunk.end_ts}] text: {chunk.text}") + + + For more information, refer to the + `Python sample `__. + + .. tab-item:: C++ + :sync: cpp + + .. code-block:: cpp + + #include "audio_utils.hpp" + #include "openvino/genai/whisper_pipeline.hpp" + + int main(int argc, char* argv[]) try { + if (3 > argc) { + throw std::runtime_error(std::string{"Usage: "} + argv[0] + " \"\""); + } + + std::filesystem::path models_path = argv[1]; + std::string wav_file_path = argv[2]; + std::string device = "CPU"; // GPU or NPU can be used as well. + + ov::genai::WhisperPipeline pipeline(models_path, device); + + ov::genai::WhisperGenerationConfig config(models_path / "generation_config.json"); + config.max_new_tokens = 100; + config.language = "<|en|>"; + config.task = "transcribe"; + config.return_timestamps = true; + + // The pipeline expects normalized audio with a sampling rate of 16kHz. + ov::genai::RawSpeechInput raw_speech = utils::audio::read_wav(wav_file_path); + auto result = pipeline.generate(raw_speech, config); + + std::cout << result << "\n"; + + for (auto& chunk : *result.chunks) { + std::cout << "timestamps: [" << chunk.start_ts << ", " << chunk.end_ts << "] text: " << chunk.text << "\n"; + } + + } catch (const std::exception& error) { + try { + std::cerr << error.what() << '\n'; + } catch (const std::ios_base::failure&) { + } + return EXIT_FAILURE; + } catch (...) { + try { + std::cerr << "Non-exception object thrown\n"; + } catch (const std::ios_base::failure&) { + } + return EXIT_FAILURE; + } + + For more information, refer to the + `C++ sample `__. + + +.. dropdown:: Using GenAI in Chat Scenario + + For chat scenarios where inputs and outputs represent a conversation, maintaining KVCache + across inputs may prove beneficial. The ``start_chat`` and ``finish_chat`` chat-specific + methods are used to mark a conversation session, as shown in the samples below: .. tab-set:: @@ -50,9 +337,35 @@ will not work with these instructions, make sure to .. code-block:: python - import openvino_genai as ov_genai - pipe = ov_genai.LLMPipeline(model_path, "CPU") - print(pipe.generate("The Sun is yellow because", max_new_tokens=100)) + import openvino_genai + + + def streamer(subword): + print(subword, end='', flush=True) + return False + + + def infer(model_dir: str): + device = 'CPU' # GPU can be used as well. + pipe = openvino_genai.LLMPipeline(model_dir, device) + + config = openvino_genai.GenerationConfig() + config.max_new_tokens = 100 + + pipe.start_chat() + while True: + try: + prompt = input('question:\n') + except EOFError: + break + pipe.generate(prompt, config, streamer) + print('\n----------') + pipe.finish_chat() + + + + For more information, refer to the + `Python sample `__. .. tab-item:: C++ :sync: cpp @@ -60,27 +373,250 @@ will not work with these instructions, make sure to .. code-block:: cpp #include "openvino/genai/llm_pipeline.hpp" - #include - int main(int argc, char* argv[]) { - std::string model_path = argv[1]; - ov::genai::LLMPipeline pipe(model_path, "CPU"); - std::cout << pipe.generate("The Sun is yellow because", ov::genai::max_new_tokens(100)); + int main(int argc, char* argv[]) try { + if (2 != argc) { + throw std::runtime_error(std::string{"Usage: "} + argv[0] + " "); + } + std::string prompt; + std::string models_path = argv[1]; + + std::string device = "CPU"; // GPU, NPU can be used as well + ov::genai::LLMPipeline pipe(models_path, device); + + ov::genai::GenerationConfig config; + config.max_new_tokens = 100; + std::function streamer = [](std::string word) { + std::cout << word << std::flush; + return false; + }; + + pipe.start_chat(); + std::cout << "question:\n"; + while (std::getline(std::cin, prompt)) { + pipe.generate(prompt, config, streamer); + std::cout << "\n----------\n" + "question:\n"; + } + pipe.finish_chat(); + } catch (const std::exception& error) { + try { + std::cerr << error.what() << '\n'; + } catch (const std::ios_base::failure&) {} + return EXIT_FAILURE; + } catch (...) { + try { + std::cerr << "Non-exception object thrown\n"; + } catch (const std::ios_base::failure&) {} + return EXIT_FAILURE; + } + + + For more information, refer to the + `C++ sample `__ + + +.. dropdown:: Using GenAI with Vision Language Models + + OpenVINO GenAI introduces the ``openvino_genai.VLMPipeline`` pipeline for + inference of multimodal text-generation Vision Language Models (VLMs). + With a text prompt and an image as input, VLMPipeline can generate text using + models such as LLava or MiniCPM-V. See the chat scenario presented + in the samples below: + + .. tab-set:: + + .. tab-item:: Python + :sync: py + + .. code-block:: python + + import numpy as np + import openvino_genai + from PIL import Image + from openvino import Tensor + from pathlib import Path + + + def streamer(subword: str) -> bool: + print(subword, end='', flush=True) + + + def read_image(path: str) -> Tensor: + pic = Image.open(path).convert("RGB") + image_data = np.array(pic.getdata()).reshape(1, pic.size[1], pic.size[0], 3).astype(np.uint8) + return Tensor(image_data) + + + def read_images(path: str) -> list[Tensor]: + entry = Path(path) + if entry.is_dir(): + return [read_image(str(file)) for file in sorted(entry.iterdir())] + return [read_image(path)] + + + def infer(model_dir: str, image_dir: str): + rgbs = read_images(image_dir) + device = 'CPU' # GPU can be used as well. + enable_compile_cache = dict() + if "GPU" == device: + enable_compile_cache["CACHE_DIR"] = "vlm_cache" + pipe = openvino_genai.VLMPipeline(model_dir, device, **enable_compile_cache) + + config = openvino_genai.GenerationConfig() + config.max_new_tokens = 100 + + pipe.start_chat() + prompt = input('question:\n') + pipe.generate(prompt, images=rgbs, generation_config=config, streamer=streamer) + + while True: + try: + prompt = input("\n----------\n" + "question:\n") + except EOFError: + break + pipe.generate(prompt, generation_config=config, streamer=streamer) + pipe.finish_chat() + + + For more information, refer to the + `Python sample `__. + + .. tab-item:: C++ + :sync: cpp + + .. code-block:: cpp + + #include "load_image.hpp" + #include + #include + + bool print_subword(std::string&& subword) { + return !(std::cout << subword << std::flush); } -The `LLMPipeline` is the main object used for decoding. You can construct it directly from the -folder with the converted model. It will automatically load the main model, tokenizer, detokenizer, -and the default generation configuration. + int main(int argc, char* argv[]) try { + if (3 != argc) { + throw std::runtime_error(std::string{"Usage "} + argv[0] + " "); + } + + std::vector rgbs = utils::load_images(argv[2]); + + std::string device = "CPU"; // GPU can be used as well. + ov::AnyMap enable_compile_cache; + if ("GPU" == device) { + enable_compile_cache.insert({ov::cache_dir("vlm_cache")}); + } + ov::genai::VLMPipeline pipe(argv[1], device, enable_compile_cache); + + ov::genai::GenerationConfig generation_config; + generation_config.max_new_tokens = 100; + + std::string prompt; + + pipe.start_chat(); + std::cout << "question:\n"; + + std::getline(std::cin, prompt); + pipe.generate(prompt, + ov::genai::images(rgbs), + ov::genai::generation_config(generation_config), + ov::genai::streamer(print_subword)); + std::cout << "\n----------\n" + "question:\n"; + while (std::getline(std::cin, prompt)) { + pipe.generate(prompt, + ov::genai::generation_config(generation_config), + ov::genai::streamer(print_subword)); + std::cout << "\n----------\n" + "question:\n"; + } + pipe.finish_chat(); + } catch (const std::exception& error) { + try { + std::cerr << error.what() << '\n'; + } catch (const std::ios_base::failure&) {} + return EXIT_FAILURE; + } catch (...) { + try { + std::cerr << "Non-exception object thrown\n"; + } catch (const std::ios_base::failure&) {} + return EXIT_FAILURE; + } + + + For more information, refer to the + `C++ sample `__ + + +| + + +Chat-bot use case - step by step +############################################################################################### + +This example will show you how to create a chat-bot functionality, using the ``ov_genai.LLMPipeline`` +and a chat-tuned TinyLlama model. Apart from the basic implementation, it provides additional +optimization methods. + +Although CPU is used as inference device in the samples below, you may choose GPU instead. +Note that tasks such as token selection, tokenization, and detokenization are always handled +by CPU only. Tokenizers, represented as a separate model, are also run on CPU. + +Running the model ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +You start with exporting an LLM model via Hugging Face Optimum-Intel. Note that the precision +of ``int4`` is used, instead of the original ``fp16``, for better performance. The weight +compression is done by NNCF at the model export stage. The exported model contains all the +information necessary for execution, including the tokenizer/detokenizer and the generation +config, ensuring that its results match those generated by Hugging Face. + +The `LLMPipeline` is the main object used for decoding and handles all the necessary steps. +You can construct it directly from the folder with the converted model. + + +.. tab-set:: + + .. tab-item:: Python + :sync: py + + .. code-block:: console + + optimum-cli export openvino --model "TinyLlama/TinyLlama-1.1B-Chat-v1.0" --weight-format int4 --trust-remote-code "TinyLlama-1.1B-Chat-v1.0" + + .. code-block:: python + + import openvino_genai as ov_genai + pipe = ov_genai.LLMPipeline(model_path, "CPU") + print(pipe.generate("The Sun is yellow because", max_new_tokens=100)) + + .. tab-item:: C++ + :sync: cpp + + .. code-block:: console + + optimum-cli export openvino --model "TinyLlama/TinyLlama-1.1B-Chat-v1.0" --weight-format int4 --trust-remote-code "TinyLlama-1.1B-Chat-v1.0" + + .. code-block:: cpp + + #include "openvino/genai/llm_pipeline.hpp" + #include + + int main(int argc, char* argv[]) { + std::string model_path = argv[1]; + ov::genai::LLMPipeline pipe(model_path, "CPU"); + std::cout << pipe.generate("The Sun is yellow because", ov::genai::max_new_tokens(100)); + } + -Once the model is exported from Hugging Face Optimum-Intel, it already contains all the information -necessary for execution, including the tokenizer/detokenizer and the generation config, ensuring that -its results match those generated by Hugging Face. Streaming the Output -########################### ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -For more interactive UIs during generation, streaming of model output tokens is supported. See the example -below, where a lambda function outputs words to the console immediately upon generation: +For more interactive UIs during generation, you can stream output tokens. In this example, a +lambda function outputs words to the console immediately upon generation: .. tab-set:: @@ -177,12 +713,10 @@ You can also create your custom streamer for more sophisticated processing: Optimizing Generation with Grouped Beam Search -####################################################### - -Leverage grouped beam search decoding and configure generation_config for better text generation -quality and efficient batch processing in GenAI applications. ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -Specify generation_config to use grouped beam search: +For better text generation quality and more efficient batch processing, specify +``generation_config`` to leverage grouped beam search decoding. .. tab-set:: @@ -219,10 +753,123 @@ Specify generation_config to use grouped beam search: } +Efficient Text Generation via Speculative Decoding +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +Speculative decoding (or assisted-generation) enables faster token generation +when an additional smaller draft model is used alongside the main model. This reduces the +number of infer requests to the main model, increasing performance. + +The draft model predicts the next K tokens one by one in an autoregressive manner. The main +model validates these predictions and corrects them if necessary - in case of +a discrepancy, the main model prediction is used. Then, the draft model acquires this token and +runs prediction of the next K tokens, thus repeating the cycle. + + +.. tab-set:: + + .. tab-item:: Python + :sync: py + + .. code-block:: python + + import openvino_genai + import queue + import threading + + def streamer(subword): + print(subword, end='', flush=True) + return False + + def infer(model_dir: str, draft_model_dir: str, prompt: str): + main_device = 'CPU' # GPU can be used as well. + draft_device = 'CPU' + + scheduler_config = openvino_genai.SchedulerConfig() + scheduler_config.cache_size = 2 + + draft_model = openvino_genai.draft_model(draft_model_dir, draft_device) + + pipe = openvino_genai.LLMPipeline(model_dir, main_device, scheduler_config=scheduler_config, draft_model=draft_model) + + config = openvino_genai.GenerationConfig() + config.max_new_tokens = 100 + config.num_assistant_tokens = 5 + + pipe.generate("The Sun is yellow because", config, streamer) + + + For more information, refer to the + `Python sample `__. + + + .. tab-item:: C++ + :sync: cpp + + .. code-block:: cpp + + #include + + #include "openvino/genai/llm_pipeline.hpp" + + int main(int argc, char* argv[]) try { + if (4 != argc) { + throw std::runtime_error(std::string{"Usage: "} + argv[0] + " ''"); + } + + ov::genai::GenerationConfig config; + config.max_new_tokens = 100; + config.num_assistant_tokens = 5; + + std::string main_model_path = argv[1]; + std::string draft_model_path = argv[2]; + std::string prompt = argv[3]; + + std::string main_device = "CPU", draft_device = "CPU"; + + ov::genai::SchedulerConfig scheduler_config; + scheduler_config.cache_size = 5; + + ov::genai::LLMPipeline pipe( + main_model_path, + main_device, + ov::genai::draft_model(draft_model_path, draft_device), + ov::genai::scheduler_config(scheduler_config)); + + auto streamer = [](std::string subword) { + std::cout << subword << std::flush; + return false; + }; + + pipe.generate("The Sun is yellow because", config, streamer); + } catch (const std::exception& error) { + try { + std::cerr << error.what() << '\n'; + } catch (const std::ios_base::failure&) {} + return EXIT_FAILURE; + } catch (...) { + try { + std::cerr << "Non-exception object thrown\n"; + } catch (const std::ios_base::failure&) {} + return EXIT_FAILURE; + } + + + For more information, refer to the + `C++ sample `__ + + + + + + + + Comparing with Hugging Face Results ####################################### -Compare and analyze results with those generated by Hugging Face models. +You can compare the results of the above example with those generated by Hugging Face models by +running the following code: .. tab-set:: @@ -250,30 +897,35 @@ Compare and analyze results with those generated by Hugging Face models. assert hf_output == ov_output -GenAI API -####################################### -OpenVINO GenAI Flavor includes the following API: -* generation_config - defines a configuration class for text generation, enabling customization of the generation process such as the maximum length of the generated text, whether to ignore end-of-sentence tokens, and the specifics of the decoding strategy (greedy, beam search, or multinomial sampling). -* llm_pipeline - provides classes and utilities for text generation, including a pipeline for processing inputs, generating text, and managing outputs with configurable options. -* streamer_base - an abstract base class for creating streamers. -* tokenizer - the tokenizer class for text encoding and decoding. +GenAI API +####################################### + +The use case described here uses the following OpenVINO GenAI API methods: +* generation_config - defines a configuration class for text generation, + enabling customization of the generation process such as the maximum length of + the generated text, whether to ignore end-of-sentence tokens, and the specifics + of the decoding strategy (greedy, beam search, or multinomial sampling). +* llm_pipeline - provides classes and utilities for processing inputs, + text generation, and managing outputs with configurable options. +* streamer_base - an abstract base class for creating streamers. +* tokenizer - the tokenizer class for text encoding and decoding. * visibility - controls the visibility of the GenAI library. -Learn more in the `GenAI API reference `__. +Learn more from the `GenAI API reference `__. Additional Resources #################### * `OpenVINO GenAI Repo `__ * `OpenVINO GenAI Samples `__ +* A Jupyter notebook demonstrating + `Visual-language assistant with MiniCPM-V2 and OpenVINO `__ * `OpenVINO Tokenizers `__ * `Neural Network Compression Framework `__ - - diff --git a/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide/genai-use-cases.rst b/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide/genai-use-cases.rst deleted file mode 100644 index 6033bd8ed96106..00000000000000 --- a/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide/genai-use-cases.rst +++ /dev/null @@ -1,426 +0,0 @@ -GenAI Use Cases -===================== - -This article provides several use case scenarios for Generative AI model -inference. The applications presented in the code samples below -only require minimal configuration, like setting an inference device. Feel free -to explore and modify the source code as you need. - - -Using GenAI for Text-to-Image Generation -######################################## - -Examples below demonstrate inference on text-to-image models, like Stable Diffusion -1.5, 2.1, and LCM, with a text prompt as input. The :ref:`main.cpp ` -sample shows basic usage of the ``Text2ImagePipeline`` pipeline. -:ref:`lora.cpp ` shows how to apply LoRA adapters to the pipeline. - - -.. tab-set:: - - .. tab-item:: Python - :sync: python - - .. tab-set:: - - .. tab-item:: main.py - :name: mainpy - - .. code-block:: python - - import openvino_genai - from PIL import Image - import numpy as np - - class Generator(openvino_genai.Generator): - def __init__(self, seed, mu=0.0, sigma=1.0): - openvino_genai.Generator.__init__(self) - np.random.seed(seed) - self.mu = mu - self.sigma = sigma - - def next(self): - return np.random.normal(self.mu, self.sigma) - - - def infer(model_dir: str, prompt: str): - device = 'CPU' # GPU can be used as well - random_generator = Generator(42) - pipe = openvino_genai.Text2ImagePipeline(model_dir, device) - image_tensor = pipe.generate( - prompt, - width=512, - height=512, - num_inference_steps=20, - num_images_per_prompt=1, - random_generator=random_generator - ) - - image = Image.fromarray(image_tensor.data[0]) - image.save("image.bmp") - - .. tab-item:: LoRA.py - :name: lorapy - - .. code-block:: python - - import openvino as ov - import openvino_genai - import numpy as np - import sys - - - class Generator(openvino_genai.Generator): - def __init__(self, seed, mu=0.0, sigma=1.0): - openvino_genai.Generator.__init__(self) - np.random.seed(seed) - self.mu = mu - self.sigma = sigma - - def next(self): - return np.random.normal(self.mu, self.sigma) - - - def image_write(path: str, image_tensor: ov.Tensor): - from PIL import Image - image = Image.fromarray(image_tensor.data[0]) - image.save(path) - - - def infer(models_path: str, prompt: str): - prompt = "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" - - device = "CPU" # GPU, NPU can be used as well - adapter_config = openvino_genai.AdapterConfig() - - for i in range(int(len(adapters) / 2)): - adapter = openvino_genai.Adapter(adapters[2 * i]) - alpha = float(adapters[2 * i + 1]) - adapter_config.add(adapter, alpha) - - pipe = openvino_genai.Text2ImagePipeline(models_path, device, adapters=adapter_config) - print("Generating image with LoRA adapters applied, resulting image will be in lora.bmp") - image = pipe.generate(prompt, - random_generator=Generator(42), - width=512, - height=896, - num_inference_steps=20) - - image_write("lora.bmp", image) - print("Generating image without LoRA adapters applied, resulting image will be in baseline.bmp") - image = pipe.generate(prompt, - adapters=openvino_genai.AdapterConfig(), - random_generator=Generator(42), - width=512, - height=896, - num_inference_steps=20 - ) - image_write("baseline.bmp", image) - - For more information, refer to the - `Python sample `__ - - .. tab-item:: C++ - :sync: cpp - - .. tab-set:: - - .. tab-item:: main.cpp - :name: maincpp - - .. code-block:: cpp - - #include "openvino/genai/text2image/pipeline.hpp" - - #include "imwrite.hpp" - - int32_t main(int32_t argc, char* argv[]) try { - OPENVINO_ASSERT(argc == 3, "Usage: ", argv[0], " ''"); - - const std::string models_path = argv[1], prompt = argv[2]; - const std::string device = "CPU"; // GPU, NPU can be used as well - - ov::genai::Text2ImagePipeline pipe(models_path, device); - ov::Tensor image = pipe.generate(prompt, - ov::genai::width(512), - ov::genai::height(512), - ov::genai::num_inference_steps(20), - ov::genai::num_images_per_prompt(1)); - - imwrite("image_%d.bmp", image, true); - - return EXIT_SUCCESS; - } catch (const std::exception& error) { - try { - std::cerr << error.what() << '\n'; - } catch (const std::ios_base::failure&) {} - return EXIT_FAILURE; - } catch (...) { - try { - std::cerr << "Non-exception object thrown\n"; - } catch (const std::ios_base::failure&) {} - return EXIT_FAILURE; - } - - .. tab-item:: LoRA.cpp - :name: loracpp - - .. code-block:: cpp - - #include "openvino/genai/text2image/pipeline.hpp" - - #include "imwrite.hpp" - - int32_t main(int32_t argc, char* argv[]) try { - OPENVINO_ASSERT(argc >= 3 && (argc - 3) % 2 == 0, "Usage: ", argv[0], " '' [ ...]]"); - - const std::string models_path = argv[1], prompt = argv[2]; - const std::string device = "CPU"; // GPU, NPU can be used as well - - ov::genai::AdapterConfig adapter_config; - for(size_t i = 0; i < (argc - 3)/2; ++i) { - ov::genai::Adapter adapter(argv[3 + 2*i]); - float alpha = std::atof(argv[3 + 2*i + 1]); - adapter_config.add(adapter, alpha); - } - - ov::genai::Text2ImagePipeline pipe(models_path, device, ov::genai::adapters(adapter_config)); - - std::cout << "Generating image with LoRA adapters applied, resulting image will be in lora.bmp\n"; - ov::Tensor image = pipe.generate(prompt, - ov::genai::random_generator(std::make_shared(42)), - ov::genai::width(512), - ov::genai::height(896), - ov::genai::num_inference_steps(20)); - imwrite("lora.bmp", image, true); - - std::cout << "Generating image without LoRA adapters applied, resulting image will be in baseline.bmp\n"; - image = pipe.generate(prompt, - ov::genai::adapters(), - ov::genai::random_generator(std::make_shared(42)), - ov::genai::width(512), - ov::genai::height(896), - ov::genai::num_inference_steps(20)); - imwrite("baseline.bmp", image, true); - - return EXIT_SUCCESS; - } catch (const std::exception& error) { - try { - std::cerr << error.what() << '\n'; - } catch (const std::ios_base::failure&) {} - return EXIT_FAILURE; - } catch (...) { - try { - std::cerr << "Non-exception object thrown\n"; - } catch (const std::ios_base::failure&) {} - return EXIT_FAILURE; - } - - - For more information, refer to the - `C++ sample `__ - - - - - -Using GenAI in Speech Recognition -################################# - - -The application, shown in code samples below, performs inference on speech -recognition Whisper Models. The samples include the ``WhisperPipeline`` class -and use audio files in WAV format at a sampling rate of 16 kHz as input. - -.. tab-set:: - - .. tab-item:: Python - :sync: cpp - - .. code-block:: python - - import openvino_genai - import librosa - - - def read_wav(filepath): - raw_speech, samplerate = librosa.load(filepath, sr=16000) - return raw_speech.tolist() - - - def infer(model_dir: str, wav_file_path: str): - device = "CPU" # GPU or NPU can be used as well. - pipe = openvino_genai.WhisperPipeline(model_dir, device) - - # The pipeline expects normalized audio with a sampling rate of 16kHz. - raw_speech = read_wav(wav_file_path) - result = pipe.generate( - raw_speech, - max_new_tokens=100, - language="<|en|>", - task="transcribe", - return_timestamps=True, - ) - - print(result) - - for chunk in result.chunks: - print(f"timestamps: [{chunk.start_ts}, {chunk.end_ts}] text: {chunk.text}") - - - For more information, refer to the - `Python sample `__. - - .. tab-item:: C++ - :sync: cpp - - .. code-block:: cpp - - #include "audio_utils.hpp" - #include "openvino/genai/whisper_pipeline.hpp" - - int main(int argc, char* argv[]) try { - if (3 > argc) { - throw std::runtime_error(std::string{"Usage: "} + argv[0] + " \"\""); - } - - std::filesystem::path models_path = argv[1]; - std::string wav_file_path = argv[2]; - std::string device = "CPU"; // GPU or NPU can be used as well. - - ov::genai::WhisperPipeline pipeline(models_path, device); - - ov::genai::WhisperGenerationConfig config(models_path / "generation_config.json"); - config.max_new_tokens = 100; - config.language = "<|en|>"; - config.task = "transcribe"; - config.return_timestamps = true; - - // The pipeline expects normalized audio with a sampling rate of 16kHz. - ov::genai::RawSpeechInput raw_speech = utils::audio::read_wav(wav_file_path); - auto result = pipeline.generate(raw_speech, config); - - std::cout << result << "\n"; - - for (auto& chunk : *result.chunks) { - std::cout << "timestamps: [" << chunk.start_ts << ", " << chunk.end_ts << "] text: " << chunk.text << "\n"; - } - - } catch (const std::exception& error) { - try { - std::cerr << error.what() << '\n'; - } catch (const std::ios_base::failure&) { - } - return EXIT_FAILURE; - } catch (...) { - try { - std::cerr << "Non-exception object thrown\n"; - } catch (const std::ios_base::failure&) { - } - return EXIT_FAILURE; - } - - - For more information, refer to the - `C++ sample `__. - - -Using GenAI in Chat Scenario -############################ - -For chat scenarios where inputs and outputs represent a conversation, maintaining KVCache across inputs -may prove beneficial. The ``start_chat`` and ``finish_chat`` chat-specific methods are used to -mark a conversation session, as shown in the samples below: - -.. tab-set:: - - .. tab-item:: Python - :sync: py - - .. code-block:: python - - import openvino_genai - - - def streamer(subword): - print(subword, end='', flush=True) - return False - - - def infer(model_dir: str): - device = 'CPU' # GPU can be used as well. - pipe = openvino_genai.LLMPipeline(model_dir, device) - - config = openvino_genai.GenerationConfig() - config.max_new_tokens = 100 - - pipe.start_chat() - while True: - try: - prompt = input('question:\n') - except EOFError: - break - pipe.generate(prompt, config, streamer) - print('\n----------') - pipe.finish_chat() - - - - For more information, refer to the - `Python sample `__. - - .. tab-item:: C++ - :sync: cpp - - .. code-block:: cpp - - #include "openvino/genai/llm_pipeline.hpp" - - int main(int argc, char* argv[]) try { - if (2 != argc) { - throw std::runtime_error(std::string{"Usage: "} + argv[0] + " "); - } - std::string prompt; - std::string models_path = argv[1]; - - std::string device = "CPU"; // GPU, NPU can be used as well - ov::genai::LLMPipeline pipe(models_path, device); - - ov::genai::GenerationConfig config; - config.max_new_tokens = 100; - std::function streamer = [](std::string word) { - std::cout << word << std::flush; - return false; - }; - - pipe.start_chat(); - std::cout << "question:\n"; - while (std::getline(std::cin, prompt)) { - pipe.generate(prompt, config, streamer); - std::cout << "\n----------\n" - "question:\n"; - } - pipe.finish_chat(); - } catch (const std::exception& error) { - try { - std::cerr << error.what() << '\n'; - } catch (const std::ios_base::failure&) {} - return EXIT_FAILURE; - } catch (...) { - try { - std::cerr << "Non-exception object thrown\n"; - } catch (const std::ios_base::failure&) {} - return EXIT_FAILURE; - } - - - For more information, refer to the - `C++ sample `__ - -Additional Resources -##################### - -* :doc:`Install OpenVINO GenAI <../../../get-started/install-openvino/install-openvino-genai>` -* `OpenVINO GenAI Repo `__ -* `OpenVINO GenAI Samples `__ -* `OpenVINO Tokenizers `__ diff --git a/docs/articles_en/learn-openvino/llm_inference_guide/llm-inference-hf.rst b/docs/articles_en/learn-openvino/llm_inference_guide/llm-inference-hf.rst index a26b670b5314d0..4fec1acd23e6a7 100644 --- a/docs/articles_en/learn-openvino/llm_inference_guide/llm-inference-hf.rst +++ b/docs/articles_en/learn-openvino/llm_inference_guide/llm-inference-hf.rst @@ -1,4 +1,4 @@ -Run LLMs with Hugging Face and Optimum Intel +Inference with Optimum Intel =============================================================================================== .. meta:: @@ -276,9 +276,10 @@ includes **Dynamic quantization** of activations of 4/8-bit quantized MatMuls an ov_config={"KV_CACHE_PRECISION": "u8", "DYNAMIC_QUANTIZATION_GROUP_SIZE": "32", "PERFORMANCE_HINT": "LATENCY"} ) -.. note:: + .. note:: + Currently, for KV-cache quantization, GPU ignores the DYNAMIC_QUANTIZATION_GROUP_SIZE property, using ``group_size = head_size``. Additionally, it does not support the ``get_state()`` and ``set_state()`` APIs when KV-cache quantization is enabled. - Currently, both Dynamic quantization and KV-cache quantization are available for CPU device. + For GPU, KV-cache quantization is enabled by default on platforms without XMX support, and can be disabled by setting KV_CACHE_PRECISION to ``undefined``. Working with Models Tuned with LoRA diff --git a/docs/articles_en/learn-openvino/llm_inference_guide/llm-inference-native-ov.rst b/docs/articles_en/learn-openvino/llm_inference_guide/llm-inference-native-ov.rst index 2476a0423e30e1..d33ae05f68f462 100644 --- a/docs/articles_en/learn-openvino/llm_inference_guide/llm-inference-native-ov.rst +++ b/docs/articles_en/learn-openvino/llm_inference_guide/llm-inference-native-ov.rst @@ -1,4 +1,4 @@ -Run LLM Inference on Native OpenVINO (not recommended) +Generative AI with Base OpenVINO (not recommended) =============================================================================================== To run Generative AI models using native OpenVINO APIs you need to follow regular diff --git a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/npu-device.rst b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/npu-device.rst index 7b135fa7ff0b14..436d383ebf787e 100644 --- a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/npu-device.rst +++ b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/npu-device.rst @@ -146,6 +146,8 @@ offer a limited set of supported OpenVINO features. ov::intel_npu::turbo ov::intel_npu::tiles ov::intel_npu::max_tiles + ov::intel_npu::bypass_umd_caching + ov::intel_npu::defer_weights_load .. tab-item:: Read-only properties @@ -168,7 +170,6 @@ offer a limited set of supported OpenVINO features. ov::intel_npu::device_alloc_mem_size ov::intel_npu::device_total_mem_size ov::intel_npu::driver_version - ov::intel_npu::bypass_umd_caching .. note:: diff --git a/docs/sphinx_setup/_static/benchmarks_files/OV-2024.5-Performance-Data.xlsx b/docs/sphinx_setup/_static/benchmarks_files/OV-2024.5-Performance-Data.xlsx index 0c29b3282790fa..e5a6a4b039b029 100644 Binary files a/docs/sphinx_setup/_static/benchmarks_files/OV-2024.5-Performance-Data.xlsx and b/docs/sphinx_setup/_static/benchmarks_files/OV-2024.5-Performance-Data.xlsx differ diff --git a/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ov.json b/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ov.json index 59e06ef51f812d..44b5b5707042df 100644 --- a/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ov.json +++ b/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ov.json @@ -13,10 +13,7 @@ "int8": 312.06, "fp16": 345.49, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -51,10 +48,7 @@ "int8": 328.55, "fp16": 285.3, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -85,14 +79,11 @@ "throughput": { "Precisions": [ { - "int4": "", - "int8": "", + "int4": 20.07, + "int8": 17.42, "fp16": "", "fp32": "", - "bf16": "", - "token_int4": 20.07, - "token_int8": 17.42, - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -117,20 +108,17 @@ "Platform": "Intel® Arc™ A-Series Graphics dGPU", "Model": "glm-4-9b-chat", "featured_SKU": true, - "whats_new_model": true, + "whats_new_model": "false", "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", + "int4": 36.48, + "int8": 27.59, "fp16": "", "fp32": "", - "bf16": "", - "token_int4": 36.48, - "token_int8": 27.59, - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -161,14 +149,11 @@ "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 42.82, + "int8": 33.97, + "fp16": 22.23, "fp32": "", - "bf16": "", - "token_int4": 42.82, - "token_int8": 33.97, - "token_fp16": 22.23 + "bf16": "" } ], "Unit": "FPS", @@ -193,20 +178,17 @@ "Platform": "Intel® Arc™ A-Series Graphics dGPU", "Model": "llama-3-8b", "featured_SKU": true, - "whats_new_model": false, + "whats_new_model": "false", "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", + "int4": 39.6, + "int8": 30.59, "fp16": "", "fp32": "", - "bf16": "", - "token_int4": 39.6, - "token_int8": 30.59, - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -237,14 +219,11 @@ "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 55.37, + "int8": 51.62, + "fp16": 35.82, "fp32": "", - "bf16": "", - "token_int4": 55.37, - "token_int8": 51.62, - "token_fp16": 35.82 + "bf16": "" } ], "Unit": "FPS", @@ -279,10 +258,7 @@ "int8": 34.84, "fp16": 19.43, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -313,14 +289,11 @@ "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 43.4, + "int8": 32.32, + "fp16": 20.91, "fp32": "", - "bf16": "", - "token_int4": 43.4, - "token_int8": 32.32, - "token_fp16": 20.91 + "bf16": "" } ], "Unit": "FPS", @@ -355,10 +328,7 @@ "int8": 2348.6, "fp16": 2074.34, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -383,20 +353,17 @@ "Platform": "Intel® Arc™ A-Series Graphics dGPU", "Model": "phi-3-mini-4k-instruct", "featured_SKU": true, - "whats_new_model": false, + "whats_new_model": true, "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 59.06, + "int8": 47.96, + "fp16": 29.29, "fp32": "", - "bf16": "", - "token_int4": 59.06, - "token_int8": 47.96, - "token_fp16": 29.29 + "bf16": "" } ], "Unit": "FPS", @@ -427,14 +394,11 @@ "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 40.48, + "int8": 32.79, + "fp16": 20.67, "fp32": "", - "bf16": "", - "token_int4": 40.48, - "token_int8": 32.79, - "token_fp16": 20.67 + "bf16": "" } ], "Unit": "FPS", @@ -469,10 +433,7 @@ "int8": 1401.85, "fp16": 1046.9, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -507,10 +468,7 @@ "int8": 112.21, "fp16": 73.01, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -545,10 +503,7 @@ "int8": 1308.1, "fp16": 1201.69, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -583,10 +538,7 @@ "int8": "", "fp16": "", "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -621,10 +573,7 @@ "int8": 517.1, "fp16": 550.33, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -650,7 +599,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -659,10 +608,7 @@ "int8": 23.3, "fp16": "", "fp32": 23.72, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -688,7 +634,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -697,10 +643,7 @@ "int8": 228.97, "fp16": "", "fp32": 219.37, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -726,7 +669,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -735,10 +678,7 @@ "int8": 59.38, "fp16": "", "fp32": 54.24, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -764,7 +704,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -773,10 +713,7 @@ "int8": 1.26, "fp16": "", "fp32": 1.08, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -802,7 +739,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -811,10 +748,7 @@ "int8": 111.92, "fp16": "", "fp32": 98.44, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -839,8 +773,8 @@ "Platform": "Intel® Atom® X6425E CPU+iGPU", "Model": "yolo11", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "whats_new_model": "false", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -849,10 +783,7 @@ "int8": "", "fp16": "", "fp32": 34.99, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -878,7 +809,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -887,10 +818,7 @@ "int8": 36.35, "fp16": "", "fp32": 33.97, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -916,7 +844,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -925,10 +853,7 @@ "int8": 7.26, "fp16": "", "fp32": 5.01, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -954,7 +879,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -963,10 +888,7 @@ "int8": 134.16, "fp16": "", "fp32": 80.45, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -992,7 +914,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1001,10 +923,7 @@ "int8": 19.87, "fp16": "", "fp32": 8.15, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -1030,7 +949,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1039,10 +958,7 @@ "int8": 0.33, "fp16": "", "fp32": 0.13, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -1068,7 +984,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1077,10 +993,7 @@ "int8": 45.84, "fp16": "", "fp32": 21.63, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -1105,8 +1018,8 @@ "Platform": "Intel® Atom® X6425E CPU-only", "Model": "yolo11", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "whats_new_model": "false", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1115,10 +1028,7 @@ "int8": "", "fp16": "", "fp32": 5.3, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -1144,7 +1054,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1153,10 +1063,7 @@ "int8": 10.31, "fp16": "", "fp32": 5.12, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -1182,7 +1089,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1191,10 +1098,7 @@ "int8": 22.02, "fp16": 25.05, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -1220,7 +1124,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1229,10 +1133,7 @@ "int8": 187.37, "fp16": 222.58, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -1258,7 +1159,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1267,10 +1168,7 @@ "int8": 48.1, "fp16": 51.68, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -1296,7 +1194,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1305,10 +1203,7 @@ "int8": 1.16, "fp16": 1.16, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -1334,7 +1229,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1343,10 +1238,7 @@ "int8": 93.36, "fp16": 95.62, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -1372,7 +1264,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1381,10 +1273,7 @@ "int8": 31.79, "fp16": 33.13, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -1410,7 +1299,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -1419,10 +1308,7 @@ "int8": 39.3, "fp16": "", "fp32": 28.97, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -1448,7 +1334,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -1457,10 +1343,7 @@ "int8": 480.45, "fp16": "", "fp32": 302.75, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -1486,7 +1369,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -1495,10 +1378,7 @@ "int8": 129.7, "fp16": "", "fp32": 54.69, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -1524,7 +1404,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -1533,10 +1413,7 @@ "int8": 2.49, "fp16": "", "fp32": 0.86, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -1562,7 +1439,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -1571,10 +1448,7 @@ "int8": 233.16, "fp16": "", "fp32": 114.81, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -1599,8 +1473,8 @@ "Platform": "Intel® Atom® x7425E CPU+iGPU", "Model": "yolo11", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "whats_new_model": "false", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -1609,10 +1483,7 @@ "int8": "", "fp16": "", "fp32": 41.37, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -1638,7 +1509,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -1647,10 +1518,7 @@ "int8": 67.73, "fp16": "", "fp32": 36.05, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -1676,7 +1544,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1685,10 +1553,7 @@ "int8": 14.29, "fp16": "", "fp32": 11.18, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -1714,7 +1579,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1723,10 +1588,7 @@ "int8": 273.98, "fp16": "", "fp32": 169.54, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -1752,7 +1614,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1761,10 +1623,7 @@ "int8": 45.27, "fp16": "", "fp32": 18.84, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -1790,7 +1649,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1799,10 +1658,7 @@ "int8": 0.76, "fp16": "", "fp32": 0.31, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -1828,7 +1684,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1837,10 +1693,7 @@ "int8": 98.2, "fp16": "", "fp32": 45.36, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -1865,8 +1718,8 @@ "Platform": "Intel® Atom® x7425E CPU-only", "Model": "yolo11", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "whats_new_model": "false", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1875,10 +1728,7 @@ "int8": "", "fp16": "", "fp32": 13.77, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -1904,7 +1754,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1913,10 +1763,7 @@ "int8": 21.58, "fp16": "", "fp32": 11.78, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -1942,7 +1789,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1951,10 +1798,7 @@ "int8": 40.0, "fp16": 34.31, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -1980,7 +1824,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -1989,10 +1833,7 @@ "int8": 414.66, "fp16": 324.8, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -2018,7 +1859,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2027,10 +1868,7 @@ "int8": 106.34, "fp16": 64.69, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -2056,7 +1894,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2065,10 +1903,7 @@ "int8": 2.16, "fp16": 1.32, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -2094,7 +1929,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2103,10 +1938,7 @@ "int8": 211.07, "fp16": 137.13, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -2132,7 +1964,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2141,10 +1973,7 @@ "int8": 60.92, "fp16": 44.64, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -2170,7 +1999,7 @@ "Model": "bert-base-cased", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -2179,10 +2008,7 @@ "int8": 45.34, "fp16": "", "fp32": 33.5, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -2208,7 +2034,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -2217,10 +2043,7 @@ "int8": 57.78, "fp16": "", "fp32": 48.75, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -2246,7 +2069,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -2255,10 +2078,7 @@ "int8": 0.56, "fp16": "", "fp32": 0.51, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -2284,7 +2104,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -2293,10 +2113,7 @@ "int8": 525.47, "fp16": "", "fp32": 392.65, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -2322,7 +2139,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -2331,10 +2148,7 @@ "int8": 197.41, "fp16": "", "fp32": 115.71, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -2360,7 +2174,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -2369,10 +2183,7 @@ "int8": 5.38, "fp16": "", "fp32": 2.71, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -2398,7 +2209,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -2407,10 +2218,7 @@ "int8": 316.13, "fp16": "", "fp32": 194.29, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -2435,8 +2243,8 @@ "Platform": "Intel® Celeron® 6305E CPU+iGPU", "Model": "yolo11", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "whats_new_model": "false", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -2445,10 +2253,7 @@ "int8": "", "fp16": "", "fp32": 80.2, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -2474,7 +2279,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -2483,10 +2288,7 @@ "int8": 114.67, "fp16": "", "fp32": 78.26, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -2512,7 +2314,7 @@ "Model": "bert-base-cased", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2521,10 +2323,7 @@ "int8": 11.77, "fp16": "", "fp32": 4.32, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -2550,7 +2349,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2559,10 +2358,7 @@ "int8": 18.94, "fp16": "", "fp32": 11.49, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -2588,7 +2384,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2597,10 +2393,7 @@ "int8": 0.17, "fp16": "", "fp32": 0.04, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -2626,7 +2419,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2635,10 +2428,7 @@ "int8": 301.05, "fp16": "", "fp32": 132.91, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -2664,7 +2454,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2673,10 +2463,7 @@ "int8": 51.66, "fp16": "", "fp32": 14.45, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -2702,7 +2489,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2711,10 +2498,7 @@ "int8": 0.89, "fp16": "", "fp32": 0.23, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -2740,7 +2524,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2749,10 +2533,7 @@ "int8": 115.03, "fp16": "", "fp32": 36.99, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -2777,8 +2558,8 @@ "Platform": "Intel® Celeron® 6305E CPU-only", "Model": "yolo11", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "whats_new_model": "false", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2787,10 +2568,7 @@ "int8": "", "fp16": "", "fp32": 11.94, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -2816,7 +2594,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2825,10 +2603,7 @@ "int8": 25.97, "fp16": "", "fp32": 9.66, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -2854,7 +2629,7 @@ "Model": "bert-base-cased", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2863,10 +2638,7 @@ "int8": 43.69, "fp16": 33.8, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -2892,7 +2664,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2901,10 +2673,7 @@ "int8": 73.58, "fp16": 58.53, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -2930,7 +2699,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2939,10 +2708,7 @@ "int8": 0.48, "fp16": 0.52, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -2968,7 +2734,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -2977,10 +2743,7 @@ "int8": 671.35, "fp16": 504.8, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -3006,7 +2769,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3015,10 +2778,7 @@ "int8": 203.17, "fp16": 118.59, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -3044,7 +2804,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3053,10 +2813,7 @@ "int8": 5.09, "fp16": 2.78, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -3082,7 +2839,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3091,10 +2848,7 @@ "int8": 396.07, "fp16": 221.18, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -3120,7 +2874,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3129,10 +2883,7 @@ "int8": 121.77, "fp16": 81.6, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -3158,7 +2909,7 @@ "Model": "bert-base-cased", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -3167,10 +2918,7 @@ "int8": 243.99, "fp16": "", "fp32": 157.96, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -3196,7 +2944,7 @@ "Model": "efficientdet-d0", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -3205,10 +2953,7 @@ "int8": 189.52, "fp16": "", "fp32": 154.61, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -3234,7 +2979,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -3243,10 +2988,7 @@ "int8": 2.45, "fp16": "", "fp32": 1.19, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -3272,7 +3014,7 @@ "Model": "mobilenet-v2", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -3281,10 +3023,7 @@ "int8": 4485.9, "fp16": "", "fp32": 2415.8, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -3310,7 +3049,7 @@ "Model": "resnet-50", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -3319,10 +3058,7 @@ "int8": 1097.16, "fp16": "", "fp32": 475.61, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -3348,7 +3084,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -3357,10 +3093,7 @@ "int8": 18.81, "fp16": "", "fp32": 9.71, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -3386,7 +3119,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -3395,10 +3128,7 @@ "int8": 1120.99, "fp16": "", "fp32": 624.14, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -3424,7 +3154,7 @@ "Model": "yolo_v8n", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -3433,10 +3163,7 @@ "int8": 374.74, "fp16": "", "fp32": 236.96, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -3462,7 +3189,7 @@ "Model": "bert-base-cased", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3471,10 +3198,7 @@ "int8": 76.15, "fp16": "", "fp32": 30.19, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -3500,7 +3224,7 @@ "Model": "efficientdet-d0", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3509,10 +3233,7 @@ "int8": 97.68, "fp16": "", "fp32": 66.63, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -3538,7 +3259,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3547,10 +3268,7 @@ "int8": 1.2, "fp16": "", "fp32": 0.3, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -3576,7 +3294,7 @@ "Model": "mobilenet-v2", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3585,10 +3303,7 @@ "int8": 1969.75, "fp16": "", "fp32": 815.83, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -3614,7 +3329,7 @@ "Model": "resnet-50", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3623,10 +3338,7 @@ "int8": 390.17, "fp16": "", "fp32": 94.82, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -3652,7 +3364,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3661,10 +3373,7 @@ "int8": 6.38, "fp16": "", "fp32": 1.6, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -3690,7 +3399,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3699,10 +3408,7 @@ "int8": 685.79, "fp16": "", "fp32": 242.78, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -3728,7 +3434,7 @@ "Model": "yolo_v8n", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3737,10 +3443,7 @@ "int8": 166.55, "fp16": "", "fp32": 64.31, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -3766,7 +3469,7 @@ "Model": "bert-base-cased", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3775,10 +3478,7 @@ "int8": 88.41, "fp16": 74.04, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -3804,7 +3504,7 @@ "Model": "efficientdet-d0", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3813,10 +3513,7 @@ "int8": 37.81, "fp16": 34.74, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -3842,19 +3539,16 @@ "Model": "llama-2-7b-chat", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", + "int8": 0.27, + "fp16": 2.55, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": 0.27, - "token_fp16": 2.55 + "bf16": "" } ], "Unit": "FPS", @@ -3880,7 +3574,7 @@ "Model": "mobilenet-v2", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3889,10 +3583,7 @@ "int8": 1966.11, "fp16": 1346.18, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -3917,20 +3608,17 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H NPU-only", "Model": "phi-3-mini-4k-instruct", "featured_SKU": true, - "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "whats_new_model": true, + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 3.7, + "int8": 0.49, + "fp16": 3.91, "fp32": "", - "bf16": "", - "token_int4": 3.7, - "token_int8": 0.49, - "token_fp16": 3.91 + "bf16": "" } ], "Unit": "FPS", @@ -3956,7 +3644,7 @@ "Model": "resnet-50", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -3965,10 +3653,7 @@ "int8": 771.23, "fp16": 382.83, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -3994,7 +3679,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4003,10 +3688,7 @@ "int8": 705.76, "fp16": 453.35, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -4032,7 +3714,7 @@ "Model": "yolo_v8n", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4041,10 +3723,7 @@ "int8": 126.18, "fp16": 129.18, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -4070,7 +3749,7 @@ "Model": "bert-base-cased", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4079,10 +3758,7 @@ "int8": 164.18, "fp16": 107.12, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -4108,7 +3784,7 @@ "Model": "efficientdet-d0", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4117,10 +3793,7 @@ "int8": 195.27, "fp16": 164.33, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -4146,19 +3819,16 @@ "Model": "gemma-2-9b", "featured_SKU": true, "whats_new_model": true, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", + "int4": 8.94, "int8": "", - "fp16": "", + "fp16": 0.94, "fp32": "", - "bf16": "", - "token_int4": 8.94, - "token_int8": "", - "token_fp16": 0.94 + "bf16": "" } ], "Unit": "FPS", @@ -4183,20 +3853,17 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", "Model": "glm-4-9b-chat", "featured_SKU": true, - "whats_new_model": true, - "PlatformType": "Client Platforms (Intel® Core™)", + "whats_new_model": "false", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 10.82, + "int8": 6.3, + "fp16": 1.1, "fp32": "", - "bf16": "", - "token_int4": 10.82, - "token_int8": 6.3, - "token_fp16": 1.1 + "bf16": "" } ], "Unit": "FPS", @@ -4222,19 +3889,16 @@ "Model": "llama-2-7b-chat", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", + "int4": 14.62, + "int8": 8.53, "fp16": "", "fp32": "", - "bf16": "", - "token_int4": 14.62, - "token_int8": 8.53, - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -4259,20 +3923,17 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", "Model": "llama-3-8b", "featured_SKU": true, - "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "whats_new_model": "false", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 14.82, + "int8": 7.84, + "fp16": 4.04, "fp32": "", - "bf16": "", - "token_int4": 14.82, - "token_int8": 7.84, - "token_fp16": 4.04 + "bf16": "" } ], "Unit": "FPS", @@ -4298,19 +3959,16 @@ "Model": "llama-3.2-3b-instruct", "featured_SKU": true, "whats_new_model": true, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 26.17, + "int8": 20.38, + "fp16": 10.76, "fp32": "", - "bf16": "", - "token_int4": 26.17, - "token_int8": 20.38, - "token_fp16": 10.76 + "bf16": "" } ], "Unit": "FPS", @@ -4336,7 +3994,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4345,10 +4003,7 @@ "int8": 2.35, "fp16": 1.58, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -4374,19 +4029,16 @@ "Model": "mistral-7b-v0.1", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", + "int4": 15.03, + "int8": 8.94, "fp16": "", "fp32": "", - "bf16": "", - "token_int4": 15.03, - "token_int8": 8.94, - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -4412,7 +4064,7 @@ "Model": "mobilenet-v2", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4421,10 +4073,7 @@ "int8": 1293.98, "fp16": 1371.59, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -4449,20 +4098,17 @@ "Platform": "Intel® Core™ Ultra 7 processor 155H iGPU-only", "Model": "phi-3-mini-4k-instruct", "featured_SKU": true, - "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "whats_new_model": true, + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 23.61, + "int8": 18.01, + "fp16": 9.36, "fp32": "", - "bf16": "", - "token_int4": 23.61, - "token_int8": 18.01, - "token_fp16": 9.36 + "bf16": "" } ], "Unit": "FPS", @@ -4488,19 +4134,16 @@ "Model": "qwen2-7b", "featured_SKU": true, "whats_new_model": true, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", + "int4": 16.68, + "int8": 9.5, "fp16": "", "fp32": "", - "bf16": "", - "token_int4": 16.68, - "token_int8": 9.5, - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -4526,7 +4169,7 @@ "Model": "resnet-50", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4535,10 +4178,7 @@ "int8": 563.96, "fp16": 416.13, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -4564,7 +4204,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4573,10 +4213,7 @@ "int8": 21.26, "fp16": 12.84, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -4602,7 +4239,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4611,10 +4248,7 @@ "int8": 1030.66, "fp16": 811.13, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -4640,7 +4274,7 @@ "Model": "yolo_v8n", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4649,10 +4283,7 @@ "int8": 403.44, "fp16": 306.22, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -4678,7 +4309,7 @@ "Model": "bert-base-cased", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -4687,10 +4318,7 @@ "int8": 223.99, "fp16": "", "fp32": 189.97, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -4716,7 +4344,7 @@ "Model": "efficientdet-d0", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -4725,10 +4353,7 @@ "int8": 174.87, "fp16": "", "fp32": 149.3, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -4754,7 +4379,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -4763,10 +4388,7 @@ "int8": 7.24, "fp16": "", "fp32": 3.52, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -4792,7 +4414,7 @@ "Model": "mobilenet-v2", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -4801,10 +4423,7 @@ "int8": 4846.91, "fp16": "", "fp32": 2888.98, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -4830,7 +4449,7 @@ "Model": "resnet-50", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -4839,10 +4458,7 @@ "int8": 1975.45, "fp16": "", "fp32": 922.35, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -4868,7 +4484,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -4877,10 +4493,7 @@ "int8": "", "fp16": "", "fp32": 20.97, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -4906,7 +4519,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -4915,10 +4528,7 @@ "int8": "", "fp16": "", "fp32": 585.46, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -4944,7 +4554,7 @@ "Model": "yolo_v8n", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -4953,10 +4563,7 @@ "int8": 343.07, "fp16": "", "fp32": 274.85, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -4982,7 +4589,7 @@ "Model": "bert-base-cased", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -4991,10 +4598,7 @@ "int8": 44.06, "fp16": "", "fp32": 16.03, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -5020,7 +4624,7 @@ "Model": "efficientdet-d0", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5029,10 +4633,7 @@ "int8": 53.32, "fp16": "", "fp32": 38.06, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -5058,7 +4659,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5067,10 +4668,7 @@ "int8": 0.65, "fp16": "", "fp32": 0.16, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -5096,7 +4694,7 @@ "Model": "mobilenet-v2", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5105,10 +4703,7 @@ "int8": 917.84, "fp16": "", "fp32": 490.87, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -5134,7 +4729,7 @@ "Model": "resnet-50", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5143,10 +4738,7 @@ "int8": 194.09, "fp16": "", "fp32": 52.09, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -5172,7 +4764,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5181,10 +4773,7 @@ "int8": 3.52, "fp16": "", "fp32": 0.87, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -5210,7 +4799,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5219,10 +4808,7 @@ "int8": 380.37, "fp16": "", "fp32": 135.96, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -5248,7 +4834,7 @@ "Model": "yolo_v8n", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5257,10 +4843,7 @@ "int8": 80.52, "fp16": "", "fp32": 34.88, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -5286,7 +4869,7 @@ "Model": "bert-base-cased", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5295,10 +4878,7 @@ "int8": 265.97, "fp16": 198.16, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -5324,7 +4904,7 @@ "Model": "efficientdet-d0", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5333,10 +4913,7 @@ "int8": 13.69, "fp16": 13.65, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -5362,19 +4939,16 @@ "Model": "llama-2-7b-chat", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", + "int8": 0.24, + "fp16": 4.4, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": 0.24, - "token_fp16": 4.4 + "bf16": "" } ], "Unit": "FPS", @@ -5400,7 +4974,7 @@ "Model": "mobilenet-v2", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5409,10 +4983,7 @@ "int8": 3799.36, "fp16": 3178.95, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -5437,20 +5008,17 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V NPU-only", "Model": "phi-3-mini-4k-instruct", "featured_SKU": true, - "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "whats_new_model": true, + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 3.31, + "int8": 0.72, + "fp16": 6.86, "fp32": "", - "bf16": "", - "token_int4": 3.31, - "token_int8": 0.72, - "token_fp16": 6.86 + "bf16": "" } ], "Unit": "FPS", @@ -5476,7 +5044,7 @@ "Model": "resnet-50", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5485,10 +5053,7 @@ "int8": 2161.26, "fp16": 948.32, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -5514,7 +5079,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5523,10 +5088,7 @@ "int8": 230.18, "fp16": 192.78, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -5552,7 +5114,7 @@ "Model": "yolo_v8n", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, NPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5561,10 +5123,7 @@ "int8": 401.12, "fp16": 497.56, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -5590,7 +5149,7 @@ "Model": "bert-base-cased", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5599,10 +5158,7 @@ "int8": 225.83, "fp16": 298.39, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -5628,7 +5184,7 @@ "Model": "efficientdet-d0", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5637,10 +5193,7 @@ "int8": 114.57, "fp16": 121.87, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -5666,19 +5219,16 @@ "Model": "gemma-2-9b", "featured_SKU": true, "whats_new_model": true, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 14.49, + "int8": 8.34, + "fp16": 0.59, "fp32": "", - "bf16": "", - "token_int4": 14.49, - "token_int8": 8.34, - "token_fp16": 0.59 + "bf16": "" } ], "Unit": "FPS", @@ -5703,20 +5253,17 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU-only", "Model": "glm-4-9b-chat", "featured_SKU": true, - "whats_new_model": true, - "PlatformType": "Client Platforms (Intel® Core™)", + "whats_new_model": "false", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 17.63, + "int8": 9.8, + "fp16": 0.71, "fp32": "", - "bf16": "", - "token_int4": 17.63, - "token_int8": 9.8, - "token_fp16": 0.71 + "bf16": "" } ], "Unit": "FPS", @@ -5742,19 +5289,16 @@ "Model": "llama-2-7b-chat", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 18.48, + "int8": 11.87, + "fp16": 6.44, "fp32": "", - "bf16": "", - "token_int4": 18.48, - "token_int8": 11.87, - "token_fp16": 6.44 + "bf16": "" } ], "Unit": "FPS", @@ -5779,20 +5323,17 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU-only", "Model": "llama-3-8b", "featured_SKU": true, - "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "whats_new_model": "false", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 20.41, + "int8": 11.07, + "fp16": 5.81, "fp32": "", - "bf16": "", - "token_int4": 20.41, - "token_int8": 11.07, - "token_fp16": 5.81 + "bf16": "" } ], "Unit": "FPS", @@ -5818,19 +5359,16 @@ "Model": "llama-3.2-3b-instruct", "featured_SKU": true, "whats_new_model": true, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 36.58, + "int8": 23.94, + "fp16": 12.86, "fp32": "", - "bf16": "", - "token_int4": 36.58, - "token_int8": 23.94, - "token_fp16": 12.86 + "bf16": "" } ], "Unit": "FPS", @@ -5856,7 +5394,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5865,10 +5403,7 @@ "int8": 10.4, "fp16": 5.7, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -5894,19 +5429,16 @@ "Model": "mistral-7b-v0.1", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 20.06, + "int8": 11.6, + "fp16": 6.05, "fp32": "", - "bf16": "", - "token_int4": 20.06, - "token_int8": 11.6, - "token_fp16": 6.05 + "bf16": "" } ], "Unit": "FPS", @@ -5932,7 +5464,7 @@ "Model": "mobilenet-v2", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -5941,10 +5473,7 @@ "int8": 1007.75, "fp16": 862.8, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -5969,20 +5498,17 @@ "Platform": "Intel® Core™ Ultra 7 processor 268V iGPU-only", "Model": "phi-3-mini-4k-instruct", "featured_SKU": true, - "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "whats_new_model": true, + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 31.27, + "int8": 20.55, + "fp16": 11.04, "fp32": "", - "bf16": "", - "token_int4": 31.27, - "token_int8": 20.55, - "token_fp16": 11.04 + "bf16": "" } ], "Unit": "FPS", @@ -6008,19 +5534,16 @@ "Model": "qwen2-7b", "featured_SKU": true, "whats_new_model": true, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 20.99, + "int8": 12.69, + "fp16": 6.07, "fp32": "", - "bf16": "", - "token_int4": 20.99, - "token_int8": 12.69, - "token_fp16": 6.07 + "bf16": "" } ], "Unit": "FPS", @@ -6046,7 +5569,7 @@ "Model": "resnet-50", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6055,10 +5578,7 @@ "int8": 830.46, "fp16": 585.38, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -6084,7 +5604,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6093,10 +5613,7 @@ "int8": 57.99, "fp16": 32.18, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -6122,7 +5639,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6131,10 +5648,7 @@ "int8": 485.85, "fp16": 555.71, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -6160,7 +5674,7 @@ "Model": "yolo_v8n", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6169,10 +5683,7 @@ "int8": 362.75, "fp16": 375.06, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -6198,7 +5709,7 @@ "Model": "bert-base-cased", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -6207,10 +5718,7 @@ "int8": 34.21, "fp16": "", "fp32": 15.71, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -6236,7 +5744,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -6245,10 +5753,7 @@ "int8": 47.95, "fp16": "", "fp32": 29.38, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -6274,7 +5779,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -6283,10 +5788,7 @@ "int8": 0.5, "fp16": "", "fp32": 0.18, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -6312,7 +5814,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -6321,10 +5823,7 @@ "int8": 742.67, "fp16": "", "fp32": 331.98, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -6350,7 +5849,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -6359,10 +5858,7 @@ "int8": 162.84, "fp16": "", "fp32": 51.66, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -6388,7 +5884,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -6397,10 +5893,7 @@ "int8": "", "fp16": "", "fp32": 1.03, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -6426,7 +5919,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -6435,10 +5928,7 @@ "int8": 328.29, "fp16": "", "fp32": 115.41, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -6463,8 +5953,8 @@ "Platform": "Intel® Core™ i5-1235U Processor CPU+iGPU", "Model": "yolo11", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "whats_new_model": "false", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -6473,10 +5963,7 @@ "int8": "", "fp16": "", "fp32": 41.68, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -6502,7 +5989,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -6511,10 +5998,7 @@ "int8": 79.4, "fp16": "", "fp32": 35.44, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -6540,7 +6024,7 @@ "Model": "bert-base-cased", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6549,10 +6033,7 @@ "int8": 31.55, "fp16": "", "fp32": 12.38, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -6578,7 +6059,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6587,10 +6068,7 @@ "int8": 43.39, "fp16": "", "fp32": 23.14, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -6616,7 +6094,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6625,10 +6103,7 @@ "int8": 0.45, "fp16": "", "fp32": 0.12, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -6654,7 +6129,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6663,10 +6138,7 @@ "int8": 789.02, "fp16": "", "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -6692,7 +6164,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6701,10 +6173,7 @@ "int8": 147.74, "fp16": "", "fp32": 38.84, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -6730,7 +6199,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6739,10 +6208,7 @@ "int8": 2.66, "fp16": "", "fp32": 0.77, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -6768,7 +6234,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6777,10 +6243,7 @@ "int8": 313.17, "fp16": "", "fp32": 95.81, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -6805,8 +6268,8 @@ "Platform": "Intel® Core™ i5-1235U Processor CPU-only", "Model": "yolo11", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "whats_new_model": "false", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6815,10 +6278,7 @@ "int8": "", "fp16": "", "fp32": 31.84, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -6844,7 +6304,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6853,10 +6313,7 @@ "int8": 67.43, "fp16": "", "fp32": 26.68, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -6882,7 +6339,7 @@ "Model": "bert-base-cased", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6891,10 +6348,7 @@ "int8": 46.15, "fp16": 38.3, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -6920,7 +6374,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6929,10 +6383,7 @@ "int8": 64.24, "fp16": 50.43, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -6958,7 +6409,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -6967,10 +6418,7 @@ "int8": 0.5, "fp16": 0.51, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -6996,7 +6444,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7005,10 +6453,7 @@ "int8": 768.31, "fp16": 485.7, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -7034,7 +6479,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7043,10 +6488,7 @@ "int8": 208.55, "fp16": 117.84, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -7072,7 +6514,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7081,10 +6523,7 @@ "int8": 5.64, "fp16": 2.72, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -7110,7 +6549,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7119,10 +6558,7 @@ "int8": 382.92, "fp16": 223.39, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -7148,7 +6584,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7157,10 +6593,7 @@ "int8": 126.83, "fp16": 77.91, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -7186,7 +6619,7 @@ "Model": "bert-base-cased", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -7195,10 +6628,7 @@ "int8": 49.68, "fp16": "", "fp32": 26.85, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -7224,7 +6654,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -7233,10 +6663,7 @@ "int8": 73.94, "fp16": "", "fp32": 48.63, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -7262,7 +6689,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -7271,10 +6698,7 @@ "int8": 0.69, "fp16": "", "fp32": 0.3, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -7300,7 +6724,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -7309,10 +6733,7 @@ "int8": 1050.26, "fp16": "", "fp32": 535.0, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -7338,7 +6759,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -7347,10 +6768,7 @@ "int8": 234.19, "fp16": "", "fp32": 87.89, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -7376,7 +6794,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -7385,10 +6803,7 @@ "int8": 4.74, "fp16": "", "fp32": 1.74, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -7414,7 +6829,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -7423,10 +6838,7 @@ "int8": 466.65, "fp16": "", "fp32": 188.83, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -7451,8 +6863,8 @@ "Platform": "Intel® Core™ i5-1335U Processor CPU+iGPU", "Model": "yolo11", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "whats_new_model": "false", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -7461,10 +6873,7 @@ "int8": "", "fp16": "", "fp32": 65.34, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -7490,7 +6899,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -7499,10 +6908,7 @@ "int8": 125.18, "fp16": "", "fp32": 58.13, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -7528,7 +6934,7 @@ "Model": "bert-base-cased", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7537,10 +6943,7 @@ "int8": 39.97, "fp16": "", "fp32": 15.97, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -7566,7 +6969,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7575,10 +6978,7 @@ "int8": 56.15, "fp16": "", "fp32": 35.76, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -7604,7 +7004,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7613,10 +7013,7 @@ "int8": 0.57, "fp16": "", "fp32": 0.16, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -7642,7 +7039,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7651,10 +7048,7 @@ "int8": 951.93, "fp16": "", "fp32": 463.06, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -7680,7 +7074,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7689,10 +7083,7 @@ "int8": 184.54, "fp16": "", "fp32": 52.88, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -7718,7 +7109,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7727,10 +7118,7 @@ "int8": 3.16, "fp16": "", "fp32": 0.92, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -7756,7 +7144,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7765,10 +7153,7 @@ "int8": 383.62, "fp16": "", "fp32": 134.93, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -7793,8 +7178,8 @@ "Platform": "Intel® Core™ i5-1335U Processor CPU-only", "Model": "yolo11", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "whats_new_model": "false", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7803,10 +7188,7 @@ "int8": "", "fp16": "", "fp32": 43.64, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -7832,7 +7214,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7841,10 +7223,7 @@ "int8": 91.3, "fp16": "", "fp32": 36.39, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -7870,7 +7249,7 @@ "Model": "bert-base-cased", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7879,10 +7258,7 @@ "int8": 47.17, "fp16": 39.79, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -7908,7 +7284,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7917,10 +7293,7 @@ "int8": 80.6, "fp16": 59.92, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -7946,7 +7319,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7955,10 +7328,7 @@ "int8": 0.52, "fp16": 0.58, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -7984,7 +7354,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -7993,10 +7363,7 @@ "int8": 778.4, "fp16": 509.56, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -8022,7 +7389,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8031,10 +7398,7 @@ "int8": 225.12, "fp16": 127.27, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -8060,7 +7424,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8069,10 +7433,7 @@ "int8": 5.79, "fp16": 2.86, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -8098,7 +7459,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8107,10 +7468,7 @@ "int8": 404.76, "fp16": 237.61, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -8136,7 +7494,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8145,10 +7503,7 @@ "int8": 131.89, "fp16": 83.17, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -8174,7 +7529,7 @@ "Model": "bert-base-cased", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8183,10 +7538,7 @@ "int8": 120.44, "fp16": "", "fp32": 47.21, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -8212,7 +7564,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8221,10 +7573,7 @@ "int8": 148.91, "fp16": "", "fp32": 93.08, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -8250,7 +7599,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8259,10 +7608,7 @@ "int8": "", "fp16": "", "fp32": 0.49, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -8288,7 +7634,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8297,10 +7643,7 @@ "int8": 2974.41, "fp16": "", "fp32": 1317.04, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -8326,7 +7669,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8335,10 +7678,7 @@ "int8": 537.98, "fp16": "", "fp32": 148.85, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -8364,7 +7704,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8373,10 +7713,7 @@ "int8": 8.8, "fp16": "", "fp32": 2.47, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -8402,7 +7739,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8411,10 +7748,7 @@ "int8": 1068.19, "fp16": "", "fp32": 379.85, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -8439,8 +7773,8 @@ "Platform": "Intel® Core™ i5-13600K CPU-only", "Model": "yolo11", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "whats_new_model": "false", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8449,10 +7783,7 @@ "int8": "", "fp16": "", "fp32": 122.62, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -8478,7 +7809,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8487,10 +7818,7 @@ "int8": 266.57, "fp16": "", "fp32": 102.14, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -8516,7 +7844,7 @@ "Model": "bert-base-cased", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -8525,10 +7853,7 @@ "int8": 84.71, "fp16": "", "fp32": 51.06, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -8554,7 +7879,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -8563,10 +7888,7 @@ "int8": 98.02, "fp16": "", "fp32": 65.51, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -8592,7 +7914,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -8601,10 +7923,7 @@ "int8": 1.16, "fp16": "", "fp32": 0.64, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -8630,7 +7949,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -8639,10 +7958,7 @@ "int8": 1353.32, "fp16": "", "fp32": 683.15, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -8668,7 +7984,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -8677,10 +7993,7 @@ "int8": 365.63, "fp16": "", "fp32": 164.12, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -8706,7 +8019,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -8715,10 +8028,7 @@ "int8": 8.65, "fp16": "", "fp32": 3.77, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -8744,7 +8054,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -8753,10 +8063,7 @@ "int8": 657.26, "fp16": "", "fp32": 293.93, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -8781,8 +8088,8 @@ "Platform": "Intel® Core™ i7-1185G7 CPU+iGPU", "Model": "yolo11", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "whats_new_model": "false", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -8791,10 +8098,7 @@ "int8": "", "fp16": "", "fp32": 107.24, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -8820,7 +8124,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -8829,10 +8133,7 @@ "int8": 182.9, "fp16": "", "fp32": 101.97, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -8858,7 +8159,7 @@ "Model": "bert-base-cased", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8867,10 +8168,7 @@ "int8": 50.21, "fp16": "", "fp32": 18.33, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -8896,7 +8194,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8905,10 +8203,7 @@ "int8": 71.27, "fp16": "", "fp32": 41.39, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -8934,7 +8229,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8943,10 +8238,7 @@ "int8": 0.71, "fp16": "", "fp32": 0.19, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -8972,7 +8264,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -8981,10 +8273,7 @@ "int8": 1291.06, "fp16": "", "fp32": 507.09, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -9010,7 +8299,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9019,10 +8308,7 @@ "int8": 224.68, "fp16": "", "fp32": 60.81, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -9048,7 +8334,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9057,10 +8343,7 @@ "int8": 3.84, "fp16": "", "fp32": 1.01, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -9086,7 +8369,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9095,10 +8378,7 @@ "int8": 491.99, "fp16": "", "fp32": 146.3, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -9123,8 +8403,8 @@ "Platform": "Intel® Core™ i7-1185G7 CPU-only", "Model": "yolo11", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "whats_new_model": "false", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9133,10 +8413,7 @@ "int8": "", "fp16": "", "fp32": 48.0, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -9162,7 +8439,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9171,10 +8448,7 @@ "int8": 106.45, "fp16": "", "fp32": 40.14, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -9200,7 +8474,7 @@ "Model": "bert-base-cased", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9209,10 +8483,7 @@ "int8": 68.4, "fp16": 53.22, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -9238,7 +8509,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9247,10 +8518,7 @@ "int8": 91.46, "fp16": 72.22, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -9276,7 +8544,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9285,10 +8553,7 @@ "int8": 0.82, "fp16": 0.88, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -9314,7 +8579,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9323,10 +8588,7 @@ "int8": 729.72, "fp16": 569.2, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -9352,7 +8614,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9361,10 +8623,7 @@ "int8": 262.94, "fp16": 174.98, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -9390,7 +8649,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9399,10 +8658,7 @@ "int8": 8.29, "fp16": 4.67, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -9428,7 +8684,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9437,10 +8693,7 @@ "int8": 447.59, "fp16": 299.29, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -9466,7 +8719,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9475,10 +8728,7 @@ "int8": 161.26, "fp16": 111.45, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -9504,7 +8754,7 @@ "Model": "bert-base-cased", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -9513,10 +8763,7 @@ "int8": 50.01, "fp16": "", "fp32": 25.82, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -9542,7 +8789,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -9551,10 +8798,7 @@ "int8": 57.69, "fp16": "", "fp32": 28.41, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -9580,7 +8824,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -9589,10 +8833,7 @@ "int8": 0.69, "fp16": "", "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -9618,7 +8859,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -9627,10 +8868,7 @@ "int8": 958.94, "fp16": "", "fp32": 350.53, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -9656,7 +8894,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -9665,10 +8903,7 @@ "int8": 230.4, "fp16": "", "fp32": 85.03, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -9694,7 +8929,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -9703,10 +8938,7 @@ "int8": 4.44, "fp16": "", "fp32": 1.75, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -9732,7 +8964,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -9741,10 +8973,7 @@ "int8": 456.16, "fp16": "", "fp32": 162.16, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -9769,8 +8998,8 @@ "Platform": "Intel® Core™ i7-1185GRE CPU+iGPU", "Model": "yolo11", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "whats_new_model": "false", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -9779,10 +9008,7 @@ "int8": "", "fp16": "", "fp32": 55.98, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -9808,7 +9034,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -9817,10 +9043,7 @@ "int8": 103.63, "fp16": "", "fp32": 53.56, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -9846,7 +9069,7 @@ "Model": "bert-base-cased", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9855,10 +9078,7 @@ "int8": 38.28, "fp16": "", "fp32": 13.87, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -9884,7 +9104,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9893,10 +9113,7 @@ "int8": 53.34, "fp16": "", "fp32": 22.26, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -9922,7 +9139,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9931,10 +9148,7 @@ "int8": 0.52, "fp16": "", "fp32": 0.14, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -9960,7 +9174,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -9969,10 +9183,7 @@ "int8": 972.25, "fp16": "", "fp32": 311.82, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -9998,7 +9209,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10007,10 +9218,7 @@ "int8": 174.69, "fp16": "", "fp32": 45.52, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -10036,7 +9244,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10045,10 +9253,7 @@ "int8": 2.72, "fp16": "", "fp32": 0.78, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -10074,7 +9279,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10083,10 +9288,7 @@ "int8": 386.67, "fp16": "", "fp32": 99.8, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -10111,8 +9313,8 @@ "Platform": "Intel® Core™ i7-1185GRE CPU-only", "Model": "yolo11", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "whats_new_model": "false", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10121,10 +9323,7 @@ "int8": "", "fp16": "", "fp32": 32.19, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -10150,7 +9349,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10159,10 +9358,7 @@ "int8": 76.54, "fp16": "", "fp32": 27.6, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -10188,7 +9384,7 @@ "Model": "bert-base-cased", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10197,10 +9393,7 @@ "int8": 45.77, "fp16": 40.93, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -10226,7 +9419,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10235,10 +9428,7 @@ "int8": 56.2, "fp16": 41.8, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -10264,7 +9454,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10273,10 +9463,7 @@ "int8": 0.56, "fp16": 0.54, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -10302,7 +9489,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10311,10 +9498,7 @@ "int8": 648.66, "fp16": 431.47, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -10340,7 +9524,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10349,10 +9533,7 @@ "int8": 208.21, "fp16": 122.24, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -10378,7 +9559,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10387,10 +9568,7 @@ "int8": 5.71, "fp16": 3.09, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -10416,7 +9594,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10425,10 +9603,7 @@ "int8": 348.95, "fp16": 224.45, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -10454,7 +9629,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10463,10 +9638,7 @@ "int8": 113.89, "fp16": 78.71, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -10492,7 +9664,7 @@ "Model": "bert-base-cased", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -10501,10 +9673,7 @@ "int8": 111.58, "fp16": "", "fp32": 57.55, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -10530,7 +9699,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -10539,10 +9708,7 @@ "int8": 141.13, "fp16": "", "fp32": 75.23, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -10568,7 +9734,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -10577,10 +9743,7 @@ "int8": 1.63, "fp16": "", "fp32": 0.68, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -10606,7 +9769,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -10615,10 +9778,7 @@ "int8": 2287.47, "fp16": "", "fp32": 1150.08, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -10644,7 +9804,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -10653,10 +9813,7 @@ "int8": 532.56, "fp16": "", "fp32": 180.65, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -10682,7 +9839,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -10691,10 +9848,7 @@ "int8": 10.33, "fp16": "", "fp32": 3.81, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -10720,7 +9874,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -10729,10 +9883,7 @@ "int8": 1013.57, "fp16": "", "fp32": 403.5, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -10757,8 +9908,8 @@ "Platform": "Intel® Core™ i7-12700H CPU+iGPU", "Model": "yolo11", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "whats_new_model": "false", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -10767,10 +9918,7 @@ "int8": "", "fp16": "", "fp32": 133.88, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -10796,7 +9944,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -10805,10 +9953,7 @@ "int8": 268.57, "fp16": "", "fp32": 120.55, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -10834,7 +9979,7 @@ "Model": "bert-base-cased", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10843,10 +9988,7 @@ "int8": 87.88, "fp16": "", "fp32": 34.76, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -10872,7 +10014,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10881,10 +10023,7 @@ "int8": 113.82, "fp16": "", "fp32": 62.45, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -10910,7 +10049,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10919,10 +10058,7 @@ "int8": 1.27, "fp16": "", "fp32": 0.36, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -10948,7 +10084,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10957,10 +10093,7 @@ "int8": 1982.75, "fp16": "", "fp32": 968.72, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -10986,7 +10119,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -10995,10 +10128,7 @@ "int8": 429.58, "fp16": "", "fp32": 107.58, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -11024,7 +10154,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11033,10 +10163,7 @@ "int8": 7.11, "fp16": "", "fp32": 1.96, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -11062,7 +10189,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11071,10 +10198,7 @@ "int8": 854.13, "fp16": "", "fp32": 289.32, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -11099,8 +10223,8 @@ "Platform": "Intel® Core™ i7-12700H CPU-only", "Model": "yolo11", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "whats_new_model": "false", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11109,10 +10233,7 @@ "int8": "", "fp16": "", "fp32": 90.72, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -11138,7 +10259,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11147,10 +10268,7 @@ "int8": 206.32, "fp16": "", "fp32": 78.09, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -11176,7 +10294,7 @@ "Model": "bert-base-cased", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11185,10 +10303,7 @@ "int8": 89.81, "fp16": 69.99, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -11214,7 +10329,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11223,10 +10338,7 @@ "int8": 128.07, "fp16": 97.39, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -11252,7 +10364,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11261,10 +10373,7 @@ "int8": 1.04, "fp16": 1.15, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -11290,7 +10399,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11299,10 +10408,7 @@ "int8": 1281.93, "fp16": 912.69, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -11328,7 +10434,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11337,10 +10443,7 @@ "int8": 381.27, "fp16": 226.42, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -11366,7 +10469,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11375,10 +10478,7 @@ "int8": 10.47, "fp16": 6.14, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -11404,7 +10504,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11413,10 +10513,7 @@ "int8": 744.92, "fp16": 407.72, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -11442,7 +10539,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11451,10 +10548,7 @@ "int8": 215.67, "fp16": 148.01, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -11480,7 +10574,7 @@ "Model": "bert-base-cased", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -11489,10 +10583,7 @@ "int8": 61.33, "fp16": "", "fp32": 32.27, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -11518,7 +10609,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -11527,10 +10618,7 @@ "int8": 88.48, "fp16": "", "fp32": 59.03, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -11556,7 +10644,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -11565,10 +10653,7 @@ "int8": 0.81, "fp16": "", "fp32": 0.43, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -11594,7 +10679,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -11603,10 +10688,7 @@ "int8": 1218.37, "fp16": "", "fp32": 644.91, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -11632,7 +10714,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -11641,10 +10723,7 @@ "int8": 284.91, "fp16": "", "fp32": 109.93, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -11670,7 +10749,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -11679,10 +10758,7 @@ "int8": 5.67, "fp16": "", "fp32": 2.15, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -11708,7 +10784,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -11717,10 +10793,7 @@ "int8": 554.73, "fp16": "", "fp32": 228.8, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -11745,8 +10818,8 @@ "Platform": "Intel® Core™ i7-1355U Processor CPU+iGPU", "Model": "yolo11", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "whats_new_model": "false", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -11755,10 +10828,7 @@ "int8": "", "fp16": "", "fp32": 80.32, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -11784,7 +10854,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -11793,10 +10863,7 @@ "int8": 154.56, "fp16": "", "fp32": 72.19, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -11822,7 +10889,7 @@ "Model": "bert-base-cased", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11831,10 +10898,7 @@ "int8": 44.62, "fp16": "", "fp32": 17.96, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -11860,7 +10924,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11869,10 +10933,7 @@ "int8": 61.85, "fp16": "", "fp32": 39.52, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -11898,7 +10959,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11907,10 +10968,7 @@ "int8": 0.64, "fp16": "", "fp32": 0.17, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -11936,7 +10994,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11945,10 +11003,7 @@ "int8": 1042.94, "fp16": "", "fp32": 515.99, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -11974,7 +11029,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -11983,10 +11038,7 @@ "int8": 203.02, "fp16": "", "fp32": 59.12, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -12012,7 +11064,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12021,10 +11073,7 @@ "int8": 3.48, "fp16": "", "fp32": 1.03, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -12050,7 +11099,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12059,10 +11108,7 @@ "int8": 422.9, "fp16": "", "fp32": 151.69, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -12087,8 +11133,8 @@ "Platform": "Intel® Core™ i7-1355U Processor CPU-only", "Model": "yolo11", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "whats_new_model": "false", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12097,10 +11143,7 @@ "int8": "", "fp16": "", "fp32": 48.93, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -12126,7 +11169,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12135,10 +11178,7 @@ "int8": 101.73, "fp16": "", "fp32": 40.76, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -12164,7 +11204,7 @@ "Model": "bert-base-cased", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12173,10 +11213,7 @@ "int8": 67.08, "fp16": 52.9, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -12202,7 +11239,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12211,10 +11248,7 @@ "int8": 98.8, "fp16": 73.53, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -12240,19 +11274,16 @@ "Model": "gemma-2-9b", "featured_SKU": false, "whats_new_model": true, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", + "int4": 6.21, + "int8": 3.88, "fp16": "", "fp32": "", - "bf16": "", - "token_int4": 6.21, - "token_int8": 3.88, - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -12277,20 +11308,17 @@ "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", "Model": "glm-4-9b-chat", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "whats_new_model": "false", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", + "int4": 7.25, + "int8": 4.27, "fp16": "", "fp32": "", - "bf16": "", - "token_int4": 7.25, - "token_int8": 4.27, - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -12316,19 +11344,16 @@ "Model": "llama-2-7b-chat", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", + "int4": 8.53, + "int8": 5.74, "fp16": "", "fp32": "", - "bf16": "", - "token_int4": 8.53, - "token_int8": 5.74, - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -12353,20 +11378,17 @@ "Platform": "Intel® Core™ i7-1355U Processor iGPU-only", "Model": "llama-3-8b", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "whats_new_model": "false", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", + "int4": 8.49, + "int8": 5.06, "fp16": "", "fp32": "", - "bf16": "", - "token_int4": 8.49, - "token_int8": 5.06, - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -12392,19 +11414,16 @@ "Model": "llama-3.2-3b-instruct", "featured_SKU": false, "whats_new_model": true, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 16.79, + "int8": 11.89, + "fp16": 6.7, "fp32": "", - "bf16": "", - "token_int4": 16.79, - "token_int8": 11.89, - "token_fp16": 6.7 + "bf16": "" } ], "Unit": "FPS", @@ -12430,7 +11449,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12439,10 +11458,7 @@ "int8": 0.73, "fp16": 0.77, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -12468,19 +11484,16 @@ "Model": "mistral-7b-v0.1", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", + "int4": 8.86, + "int8": 5.44, "fp16": "", "fp32": "", - "bf16": "", - "token_int4": 8.86, - "token_int8": 5.44, - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -12506,7 +11519,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12515,10 +11528,7 @@ "int8": 869.88, "fp16": 621.94, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -12544,19 +11554,16 @@ "Model": "phi-3-mini-4k-instruct", "featured_SKU": false, "whats_new_model": true, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 14.78, + "int8": 9.98, + "fp16": 5.45, "fp32": "", - "bf16": "", - "token_int4": 14.78, - "token_int8": 9.98, - "token_fp16": 5.45 + "bf16": "" } ], "Unit": "FPS", @@ -12582,19 +11589,16 @@ "Model": "qwen2-7b", "featured_SKU": false, "whats_new_model": true, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", + "int4": 9.11, + "int8": 5.39, "fp16": "", "fp32": "", - "bf16": "", - "token_int4": 9.11, - "token_int8": 5.39, - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -12620,7 +11624,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12629,10 +11633,7 @@ "int8": 277.06, "fp16": 164.27, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -12658,7 +11659,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12667,10 +11668,7 @@ "int8": 7.1, "fp16": 3.99, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -12696,7 +11694,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12705,10 +11703,7 @@ "int8": 484.13, "fp16": 298.47, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -12734,7 +11729,7 @@ "Model": "stable-diffusion-v1-5", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12743,10 +11738,7 @@ "int8": "", "fp16": "", "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -12772,7 +11764,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12781,10 +11773,7 @@ "int8": 162.35, "fp16": 106.83, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -12810,7 +11799,7 @@ "Model": "bert-base-cased", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12819,10 +11808,7 @@ "int8": 170.14, "fp16": "", "fp32": 67.07, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -12848,7 +11834,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -12857,10 +11843,7 @@ "int8": 219.8, "fp16": "", "fp32": 126.91, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -12886,19 +11869,16 @@ "Model": "gemma-2-9b", "featured_SKU": false, "whats_new_model": true, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 9.43, + "int8": 6.9, + "fp16": 3.59, "fp32": "", - "bf16": "", - "token_int4": 9.42, - "token_int8": 6.89, - "token_fp16": 3.59 + "bf16": "" } ], "Unit": "FPS", @@ -12908,7 +11888,7 @@ "Precisions": [ { "int4": 106.06, - "int8": 144.95, + "int8": 144.96, "fp16": 278.42, "fp32": "", "bf16": "" @@ -12923,20 +11903,17 @@ "Platform": "Intel® Core™ i9-13900K CPU-only", "Model": "glm-4-9b-chat", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "whats_new_model": "false", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 10.66, + "int8": 7.47, + "fp16": 3.84, "fp32": "", - "bf16": "", - "token_int4": 10.65, - "token_int8": 7.46, - "token_fp16": 3.83 + "bf16": "" } ], "Unit": "FPS", @@ -12947,7 +11924,7 @@ { "int4": 93.82, "int8": 133.88, - "fp16": 260.66, + "fp16": 260.67, "fp32": "", "bf16": "" } @@ -12962,19 +11939,16 @@ "Model": "llama-2-7b-chat", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 13.44, + "int8": 9.29, + "fp16": 4.94, "fp32": "", - "bf16": "", - "token_int4": 13.44, - "token_int8": 9.29, - "token_fp16": 4.94 + "bf16": "" } ], "Unit": "FPS", @@ -12999,20 +11973,17 @@ "Platform": "Intel® Core™ i9-13900K CPU-only", "Model": "llama-3-8b", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "whats_new_model": "false", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 11.91, + "int8": 8.66, + "fp16": 4.81, "fp32": "", - "bf16": "", - "token_int4": 11.91, - "token_int8": 8.65, - "token_fp16": 4.48 + "bf16": "" } ], "Unit": "FPS", @@ -13022,7 +11993,7 @@ "Precisions": [ { "int4": 83.93, - "int8": 115.48, + "int8": 115.49, "fp16": 223.15, "fp32": "", "bf16": "" @@ -13038,19 +12009,16 @@ "Model": "llama-3.2-3b-instruct", "featured_SKU": false, "whats_new_model": true, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 25.41, + "int8": 19.0, + "fp16": 10.18, "fp32": "", - "bf16": "", - "token_int4": 25.41, - "token_int8": 18.99, - "token_fp16": 10.18 + "bf16": "" } ], "Unit": "FPS", @@ -13059,9 +12027,9 @@ "latency": { "Precisions": [ { - "int4": 39.35, + "int4": 39.36, "int8": 52.64, - "fp16": 98.23, + "fp16": 98.24, "fp32": "", "bf16": "" } @@ -13076,7 +12044,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13085,10 +12053,7 @@ "int8": 2.49, "fp16": "", "fp32": 0.71, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -13114,19 +12079,16 @@ "Model": "mistral-7b-v0.1", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 14.41, + "int8": 9.13, + "fp16": 4.72, "fp32": "", - "bf16": "", - "token_int4": 14.41, - "token_int8": 9.12, - "token_fp16": 4.71 + "bf16": "" } ], "Unit": "FPS", @@ -13135,9 +12097,9 @@ "latency": { "Precisions": [ { - "int4": 69.39, + "int4": 69.4, "int8": 109.54, - "fp16": 211.91, + "fp16": 211.92, "fp32": "", "bf16": "" } @@ -13152,7 +12114,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13161,10 +12123,7 @@ "int8": 4239.14, "fp16": "", "fp32": 2047.2, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -13190,19 +12149,16 @@ "Model": "phi-3-mini-4k-instruct", "featured_SKU": false, "whats_new_model": true, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", + "int8": 15.66, + "fp16": 8.52, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": 15.66, - "token_fp16": 8.52 + "bf16": "" } ], "Unit": "FPS", @@ -13212,7 +12168,7 @@ "Precisions": [ { "int4": "", - "int8": 63.84, + "int8": 63.85, "fp16": 117.37, "fp32": "", "bf16": "" @@ -13228,19 +12184,16 @@ "Model": "qwen2-7b", "featured_SKU": false, "whats_new_model": true, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 13.1, + "int8": 9.24, + "fp16": 4.75, "fp32": "", - "bf16": "", - "token_int4": 13.1, - "token_int8": 9.24, - "token_fp16": 4.75 + "bf16": "" } ], "Unit": "FPS", @@ -13249,9 +12202,9 @@ "latency": { "Precisions": [ { - "int4": 76.33, - "int8": 108.16, - "fp16": 210.38, + "int4": "", + "int8": 63.85, + "fp16": 117.37, "fp32": "", "bf16": "" } @@ -13266,7 +12219,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13275,10 +12228,7 @@ "int8": 762.32, "fp16": "", "fp32": 234.53, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -13304,7 +12254,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13313,10 +12263,7 @@ "int8": 12.97, "fp16": "", "fp32": 3.84, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -13342,7 +12289,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13351,10 +12298,7 @@ "int8": 1606.89, "fp16": "", "fp32": 589.62, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -13380,7 +12324,7 @@ "Model": "stable-diffusion-v1-5", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13389,10 +12333,7 @@ "int8": "", "fp16": "", "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -13417,8 +12358,8 @@ "Platform": "Intel® Core™ i9-13900K CPU-only", "Model": "yolo11", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "whats_new_model": "false", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13427,10 +12368,7 @@ "int8": "", "fp16": "", "fp32": 187.66, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -13456,7 +12394,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Client Platforms (Intel® Core™)", + "PlatformType": "Intel® Core™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -13465,10 +12403,7 @@ "int8": 389.04, "fp16": "", "fp32": 154.4, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -13492,7 +12427,7 @@ { "Platform": "Intel® Data Center GPU Flex 170 dGPU", "Model": "bert-base-cased", - "featured_SKU": false, + "featured_SKU": "false", "whats_new_model": false, "PlatformType": "Accelerator Platforms", "Parameters": { @@ -13503,10 +12438,7 @@ "int8": 385.87, "fp16": 420.99, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -13530,7 +12462,7 @@ { "Platform": "Intel® Data Center GPU Flex 170 dGPU", "Model": "efficientdet-d0", - "featured_SKU": false, + "featured_SKU": "false", "whats_new_model": false, "PlatformType": "Accelerator Platforms", "Parameters": { @@ -13541,10 +12473,7 @@ "int8": 426.56, "fp16": 362.73, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -13568,21 +12497,18 @@ { "Platform": "Intel® Data Center GPU Flex 170 dGPU", "Model": "gemma-2-9b", - "featured_SKU": false, + "featured_SKU": "false", "whats_new_model": true, "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", + "int4": 22.66, + "int8": 18.13, "fp16": "", "fp32": "", - "bf16": "", - "token_int4": 22.66, - "token_int8": 18.13, - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -13606,21 +12532,18 @@ { "Platform": "Intel® Data Center GPU Flex 170 dGPU", "Model": "glm-4-9b-chat", - "featured_SKU": false, - "whats_new_model": false, + "featured_SKU": "false", + "whats_new_model": "false", "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", + "int4": 40.04, + "int8": 26.95, "fp16": "", "fp32": "", - "bf16": "", - "token_int4": 40.04, - "token_int8": 26.95, - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -13644,21 +12567,18 @@ { "Platform": "Intel® Data Center GPU Flex 170 dGPU", "Model": "llama-2-7b-chat", - "featured_SKU": false, + "featured_SKU": "false", "whats_new_model": false, "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 45.22, + "int8": 33.88, + "fp16": 21.45, "fp32": "", - "bf16": "", - "token_int4": 45.22, - "token_int8": 33.88, - "token_fp16": 21.45 + "bf16": "" } ], "Unit": "FPS", @@ -13682,21 +12602,18 @@ { "Platform": "Intel® Data Center GPU Flex 170 dGPU", "Model": "llama-3-8b", - "featured_SKU": false, - "whats_new_model": false, + "featured_SKU": "false", + "whats_new_model": "false", "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", + "int4": 45.55, + "int8": 30.8, "fp16": "", "fp32": "", - "bf16": "", - "token_int4": 45.55, - "token_int8": 30.8, - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -13720,21 +12637,18 @@ { "Platform": "Intel® Data Center GPU Flex 170 dGPU", "Model": "llama-3.2-3b-instruct", - "featured_SKU": false, + "featured_SKU": "false", "whats_new_model": true, "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 69.44, + "int8": 57.9, + "fp16": 37.69, "fp32": "", - "bf16": "", - "token_int4": 69.44, - "token_int8": 57.9, - "token_fp16": 37.69 + "bf16": "" } ], "Unit": "FPS", @@ -13758,7 +12672,7 @@ { "Platform": "Intel® Data Center GPU Flex 170 dGPU", "Model": "mask_rcnn_resnet50_atrous_coco", - "featured_SKU": false, + "featured_SKU": "false", "whats_new_model": false, "PlatformType": "Accelerator Platforms", "Parameters": { @@ -13769,10 +12683,7 @@ "int8": 33.38, "fp16": 19.04, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -13796,21 +12707,18 @@ { "Platform": "Intel® Data Center GPU Flex 170 dGPU", "Model": "mistral-7b-v0.1", - "featured_SKU": false, + "featured_SKU": "false", "whats_new_model": false, "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 45.53, + "int8": 32.37, + "fp16": 20.21, "fp32": "", - "bf16": "", - "token_int4": 45.53, - "token_int8": 32.37, - "token_fp16": 20.21 + "bf16": "" } ], "Unit": "FPS", @@ -13834,7 +12742,7 @@ { "Platform": "Intel® Data Center GPU Flex 170 dGPU", "Model": "mobilenet-v2", - "featured_SKU": false, + "featured_SKU": "false", "whats_new_model": false, "PlatformType": "Accelerator Platforms", "Parameters": { @@ -13845,10 +12753,7 @@ "int8": 3134.27, "fp16": 3004.5, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -13872,21 +12777,18 @@ { "Platform": "Intel® Data Center GPU Flex 170 dGPU", "Model": "phi-3-mini-4k-instruct", - "featured_SKU": false, + "featured_SKU": "false", "whats_new_model": true, "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 69.93, + "int8": 51.51, + "fp16": 32.84, "fp32": "", - "bf16": "", - "token_int4": 69.93, - "token_int8": 51.51, - "token_fp16": 32.84 + "bf16": "" } ], "Unit": "FPS", @@ -13910,21 +12812,18 @@ { "Platform": "Intel® Data Center GPU Flex 170 dGPU", "Model": "qwen2-7b", - "featured_SKU": false, + "featured_SKU": "false", "whats_new_model": true, "PlatformType": "Accelerator Platforms", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", + "int4": 45.8, + "int8": 32.78, "fp16": "", "fp32": "", - "bf16": "", - "token_int4": 45.8, - "token_int8": 32.78, - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -13948,7 +12847,7 @@ { "Platform": "Intel® Data Center GPU Flex 170 dGPU", "Model": "resnet-50", - "featured_SKU": false, + "featured_SKU": "false", "whats_new_model": false, "PlatformType": "Accelerator Platforms", "Parameters": { @@ -13959,10 +12858,7 @@ "int8": 1921.18, "fp16": 1329.28, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -13986,7 +12882,7 @@ { "Platform": "Intel® Data Center GPU Flex 170 dGPU", "Model": "ssd-resnet34-1200", - "featured_SKU": false, + "featured_SKU": "false", "whats_new_model": false, "PlatformType": "Accelerator Platforms", "Parameters": { @@ -13997,10 +12893,7 @@ "int8": 133.77, "fp16": "", "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -14024,7 +12917,7 @@ { "Platform": "Intel® Data Center GPU Flex 170 dGPU", "Model": "ssd_mobilenet_v1_coco", - "featured_SKU": false, + "featured_SKU": "false", "whats_new_model": false, "PlatformType": "Accelerator Platforms", "Parameters": { @@ -14035,10 +12928,7 @@ "int8": 2200.83, "fp16": 1665.15, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -14062,7 +12952,7 @@ { "Platform": "Intel® Data Center GPU Flex 170 dGPU", "Model": "stable-diffusion-v1-5", - "featured_SKU": false, + "featured_SKU": "false", "whats_new_model": false, "PlatformType": "Accelerator Platforms", "Parameters": { @@ -14073,10 +12963,7 @@ "int8": "", "fp16": "", "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -14100,7 +12987,7 @@ { "Platform": "Intel® Data Center GPU Flex 170 dGPU", "Model": "yolo_v8n", - "featured_SKU": false, + "featured_SKU": "false", "whats_new_model": false, "PlatformType": "Accelerator Platforms", "Parameters": { @@ -14111,10 +12998,7 @@ "int8": 759.93, "fp16": 694.57, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -14140,7 +13024,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -14149,10 +13033,7 @@ "int8": 36.93, "fp16": "", "fp32": 27.64, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -14178,7 +13059,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -14187,10 +13068,7 @@ "int8": 484.32, "fp16": "", "fp32": 278.4, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -14216,7 +13094,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -14225,10 +13103,7 @@ "int8": 112.23, "fp16": "", "fp32": 42.14, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -14254,7 +13129,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -14263,10 +13138,7 @@ "int8": 2.04, "fp16": "", "fp32": 0.6, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -14292,7 +13164,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -14301,10 +13173,7 @@ "int8": 216.96, "fp16": "", "fp32": 94.92, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -14329,8 +13198,8 @@ "Platform": "Intel® Processor N100 CPU+iGPU", "Model": "yolo11", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "whats_new_model": "false", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -14339,10 +13208,7 @@ "int8": "", "fp16": "", "fp32": 34.52, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -14368,7 +13234,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU+iGPU", "Parameters": { "throughput": { "Precisions": [ @@ -14377,10 +13243,7 @@ "int8": 61.06, "fp16": "", "fp32": 28.61, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -14406,7 +13269,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14415,10 +13278,7 @@ "int8": 15.44, "fp16": "", "fp32": 12.75, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -14444,7 +13304,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14453,10 +13313,7 @@ "int8": 296.53, "fp16": "", "fp32": 183.3, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -14482,7 +13339,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14491,10 +13348,7 @@ "int8": 48.77, "fp16": "", "fp32": 20.13, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -14520,7 +13374,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14529,10 +13383,7 @@ "int8": 0.82, "fp16": "", "fp32": 0.31, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -14558,7 +13409,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14567,10 +13418,7 @@ "int8": 106.12, "fp16": "", "fp32": 49.52, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -14595,8 +13443,8 @@ "Platform": "Intel® Processor N100 CPU-only", "Model": "yolo11", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "whats_new_model": "false", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14605,10 +13453,7 @@ "int8": "", "fp16": "", "fp32": 15.36, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -14634,7 +13479,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14643,10 +13488,7 @@ "int8": 23.65, "fp16": "", "fp32": 12.86, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -14672,7 +13514,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14681,10 +13523,7 @@ "int8": 33.69, "fp16": 30.91, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -14710,7 +13549,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14719,10 +13558,7 @@ "int8": 337.95, "fp16": 267.38, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -14748,7 +13584,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14757,10 +13593,7 @@ "int8": 81.72, "fp16": 49.76, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -14786,7 +13619,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14795,10 +13628,7 @@ "int8": 1.62, "fp16": 1.01, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -14824,7 +13654,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14833,10 +13663,7 @@ "int8": 164.31, "fp16": 106.85, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -14862,7 +13689,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Mobile Platforms (Intel® Atom™)", + "PlatformType": "Intel® Atom™, iGPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14871,10 +13698,7 @@ "int8": 47.04, "fp16": 34.97, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -14900,7 +13724,7 @@ "Model": "bert-base-cased", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14909,10 +13733,7 @@ "int8": 218.18, "fp16": "", "fp32": 80.36, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -14938,7 +13759,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14947,10 +13768,7 @@ "int8": 271.94, "fp16": "", "fp32": 167.25, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -14976,7 +13794,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -14985,10 +13803,7 @@ "int8": 3.26, "fp16": "", "fp32": 0.9, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -15014,7 +13829,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15023,10 +13838,7 @@ "int8": 5417.98, "fp16": "", "fp32": 1926.0, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -15052,7 +13864,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15061,10 +13873,7 @@ "int8": 979.5, "fp16": "", "fp32": 267.16, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -15090,7 +13899,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15099,10 +13908,7 @@ "int8": 17.65, "fp16": "", "fp32": 4.58, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -15128,7 +13934,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15137,10 +13943,7 @@ "int8": 2104.85, "fp16": "", "fp32": 639.65, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -15165,8 +13968,8 @@ "Platform": "Intel® Xeon® Gold 5218T CPU-only", "Model": "yolo11", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "whats_new_model": "false", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15175,10 +13978,7 @@ "int8": "", "fp16": "", "fp32": 206.18, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -15204,7 +14004,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15213,10 +14013,7 @@ "int8": 440.56, "fp16": "", "fp32": 173.57, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -15242,7 +14039,7 @@ "Model": "bert-base-cased", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15251,10 +14048,7 @@ "int8": 426.19, "fp16": "", "fp32": 162.63, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -15280,7 +14074,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15289,10 +14083,7 @@ "int8": 411.51, "fp16": "", "fp32": 254.65, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -15318,7 +14109,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15327,10 +14118,7 @@ "int8": 6.45, "fp16": "", "fp32": 1.65, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -15356,7 +14144,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15365,10 +14153,7 @@ "int8": 10273.19, "fp16": "", "fp32": 3342.96, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -15394,7 +14179,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15403,10 +14188,7 @@ "int8": 2125.81, "fp16": "", "fp32": 570.61, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -15432,7 +14214,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15441,10 +14223,7 @@ "int8": 41.83, "fp16": "", "fp32": 10.91, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -15470,7 +14249,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15479,10 +14258,7 @@ "int8": 4376.71, "fp16": "", "fp32": 1244.57, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -15507,8 +14283,8 @@ "Platform": "Intel® Xeon® Gold 6238L CPU-only", "Model": "yolo11", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "whats_new_model": "false", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15517,10 +14293,7 @@ "int8": "", "fp16": "", "fp32": 383.86, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -15546,7 +14319,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15555,10 +14328,7 @@ "int8": 749.14, "fp16": "", "fp32": 338.04, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -15584,7 +14354,7 @@ "Model": "bert-base-cased", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15593,10 +14363,7 @@ "int8": 622.71, "fp16": "", "fp32": 240.52, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -15622,7 +14389,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15631,10 +14398,7 @@ "int8": 721.9, "fp16": "", "fp32": 423.3, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -15660,7 +14424,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15669,10 +14433,7 @@ "int8": 10.46, "fp16": "", "fp32": 2.45, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -15698,7 +14459,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15707,10 +14468,7 @@ "int8": 16509.95, "fp16": "", "fp32": 5201.56, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -15736,7 +14494,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15745,10 +14503,7 @@ "int8": 3352.09, "fp16": "", "fp32": 825.5, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -15774,7 +14529,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15783,10 +14538,7 @@ "int8": 60.91, "fp16": "", "fp32": 15.11, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -15812,7 +14564,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15821,10 +14573,7 @@ "int8": 6975.09, "fp16": "", "fp32": 1755.62, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -15849,8 +14598,8 @@ "Platform": "Intel® Xeon® Gold 6338N CPU-only", "Model": "yolo11", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "whats_new_model": "false", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15859,10 +14608,7 @@ "int8": "", "fp16": "", "fp32": 571.3, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -15888,7 +14634,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15897,10 +14643,7 @@ "int8": 1224.86, "fp16": "", "fp32": 495.73, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -15926,7 +14669,7 @@ "Model": "bert-base-cased", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15935,10 +14678,7 @@ "int8": 587.54, "fp16": "", "fp32": 225.64, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -15964,7 +14704,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -15973,10 +14713,7 @@ "int8": 580.8, "fp16": "", "fp32": 343.39, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -16002,7 +14739,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16011,10 +14748,7 @@ "int8": 8.58, "fp16": "", "fp32": 2.26, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -16040,7 +14774,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16049,10 +14783,7 @@ "int8": 14930.31, "fp16": "", "fp32": 4646.16, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -16078,7 +14809,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16087,10 +14818,7 @@ "int8": 2965.31, "fp16": "", "fp32": 761.01, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -16116,7 +14844,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16125,10 +14853,7 @@ "int8": 58.15, "fp16": "", "fp32": 15.0, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -16154,7 +14879,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16163,10 +14888,7 @@ "int8": 6130.48, "fp16": "", "fp32": 1654.84, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -16191,8 +14913,8 @@ "Platform": "Intel® Xeon® Platinum 8280 CPU-only", "Model": "yolo11", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "whats_new_model": "false", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16201,10 +14923,7 @@ "int8": "", "fp16": "", "fp32": 512.57, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -16230,7 +14949,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16239,10 +14958,7 @@ "int8": 996.59, "fp16": "", "fp32": 452.05, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -16268,7 +14984,7 @@ "Model": "bert-base-cased", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16277,10 +14993,7 @@ "int8": 881.04, "fp16": "", "fp32": 338.12, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -16306,7 +15019,7 @@ "Model": "efficientdet-d0", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16315,10 +15028,7 @@ "int8": 1009.71, "fp16": "", "fp32": 562.38, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -16344,19 +15054,16 @@ "Model": "gemma-2-9b", "featured_SKU": false, "whats_new_model": true, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 20.78, + "int8": 14.18, + "fp16": 7.72, "fp32": "", - "bf16": "", - "token_int4": 20.78, - "token_int8": 14.18, - "token_fp16": 7.72 + "bf16": "" } ], "Unit": "FPS", @@ -16381,20 +15088,17 @@ "Platform": "Intel® Xeon® Platinum 8380 CPU-only", "Model": "glm-4-9b-chat", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "whats_new_model": "false", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 22.79, + "int8": 15.56, + "fp16": 8.48, "fp32": "", - "bf16": "", - "token_int4": 22.79, - "token_int8": 15.56, - "token_fp16": 8.48 + "bf16": "" } ], "Unit": "FPS", @@ -16420,19 +15124,16 @@ "Model": "llama-2-7b-chat", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 25.41, + "int8": 18.68, + "fp16": 10.61, "fp32": "", - "bf16": "", - "token_int4": 25.41, - "token_int8": 18.68, - "token_fp16": 10.61 + "bf16": "" } ], "Unit": "FPS", @@ -16457,20 +15158,17 @@ "Platform": "Intel® Xeon® Platinum 8380 CPU-only", "Model": "llama-3-8b", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "whats_new_model": "false", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 26.07, + "int8": 17.66, + "fp16": 9.72, "fp32": "", - "bf16": "", - "token_int4": 26.07, - "token_int8": 17.66, - "token_fp16": 9.72 + "bf16": "" } ], "Unit": "FPS", @@ -16496,19 +15194,16 @@ "Model": "llama-3.2-3b-instruct", "featured_SKU": false, "whats_new_model": true, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 46.81, + "int8": 33.54, + "fp16": 19.32, "fp32": "", - "bf16": "", - "token_int4": 46.81, - "token_int8": 33.54, - "token_fp16": 19.32 + "bf16": "" } ], "Unit": "FPS", @@ -16534,7 +15229,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16543,10 +15238,7 @@ "int8": 14.73, "fp16": "", "fp32": 3.42, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -16572,19 +15264,16 @@ "Model": "mistral-7b-v0.1", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 26.89, + "int8": 18.54, + "fp16": 10.22, "fp32": "", - "bf16": "", - "token_int4": 26.89, - "token_int8": 18.54, - "token_fp16": 10.22 + "bf16": "" } ], "Unit": "FPS", @@ -16610,7 +15299,7 @@ "Model": "mobilenet-v2", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16619,10 +15308,7 @@ "int8": 22703.47, "fp16": "", "fp32": 6937.71, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -16648,19 +15334,16 @@ "Model": "phi-3-mini-4k-instruct", "featured_SKU": false, "whats_new_model": true, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 39.41, + "int8": 29.28, + "fp16": 17.35, "fp32": "", - "bf16": "", - "token_int4": 39.41, - "token_int8": 29.28, - "token_fp16": 17.35 + "bf16": "" } ], "Unit": "FPS", @@ -16686,19 +15369,16 @@ "Model": "qwen2-7b", "featured_SKU": false, "whats_new_model": true, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 28.26, + "int8": 19.32, + "fp16": 10.27, "fp32": "", - "bf16": "", - "token_int4": 28.26, - "token_int8": 19.32, - "token_fp16": 10.27 + "bf16": "" } ], "Unit": "FPS", @@ -16724,7 +15404,7 @@ "Model": "resnet-50", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16733,10 +15413,7 @@ "int8": 4874.95, "fp16": "", "fp32": 1144.73, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -16762,7 +15439,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16771,10 +15448,7 @@ "int8": 84.6, "fp16": "", "fp32": 20.95, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -16800,7 +15474,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16809,10 +15483,7 @@ "int8": 10174.18, "fp16": "", "fp32": 2524.59, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -16838,7 +15509,7 @@ "Model": "stable-diffusion-v1-5", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16847,10 +15518,7 @@ "int8": "", "fp16": "", "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -16875,8 +15543,8 @@ "Platform": "Intel® Xeon® Platinum 8380 CPU-only", "Model": "yolo11", "featured_SKU": false, - "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "whats_new_model": "false", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16885,10 +15553,7 @@ "int8": "", "fp16": "", "fp32": 803.12, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -16914,7 +15579,7 @@ "Model": "yolo_v8n", "featured_SKU": false, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16923,10 +15588,7 @@ "int8": 1704.08, "fp16": "", "fp32": 697.23, - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -16952,7 +15614,7 @@ "Model": "bert-base-cased", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16961,10 +15623,7 @@ "int8": 3023.92, "fp16": "", "fp32": 483.11, - "bf16": 1976.63, - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": 1976.63 } ], "Unit": "FPS", @@ -16990,7 +15649,7 @@ "Model": "efficientdet-d0", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -16999,10 +15658,7 @@ "int8": 1445.78, "fp16": "", "fp32": 861.51, - "bf16": 1021.75, - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": 1021.75 } ], "Unit": "FPS", @@ -17028,19 +15684,16 @@ "Model": "gemma-2-9b", "featured_SKU": true, "whats_new_model": true, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 22.71, + "int8": 16.83, + "fp16": 10.76, "fp32": "", - "bf16": "", - "token_int4": 22.71, - "token_int8": 16.83, - "token_fp16": 10.76 + "bf16": "" } ], "Unit": "FPS", @@ -17065,20 +15718,17 @@ "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", "Model": "glm-4-9b-chat", "featured_SKU": true, - "whats_new_model": true, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "whats_new_model": "false", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 23.7, + "int8": 16.93, + "fp16": 11.27, "fp32": "", - "bf16": "", - "token_int4": 23.7, - "token_int8": 16.93, - "token_fp16": 11.27 + "bf16": "" } ], "Unit": "FPS", @@ -17104,19 +15754,16 @@ "Model": "llama-2-7b-chat", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 26.11, + "int8": 20.1, + "fp16": 14.19, "fp32": "", - "bf16": "", - "token_int4": 26.11, - "token_int8": 20.1, - "token_fp16": 14.19 + "bf16": "" } ], "Unit": "FPS", @@ -17141,20 +15788,17 @@ "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", "Model": "llama-3-8b", "featured_SKU": true, - "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "whats_new_model": "false", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 26.02, + "int8": 18.97, + "fp16": 13.23, "fp32": "", - "bf16": "", - "token_int4": 26.02, - "token_int8": 18.97, - "token_fp16": 13.23 + "bf16": "" } ], "Unit": "FPS", @@ -17180,19 +15824,16 @@ "Model": "llama-3.2-3b-instruct", "featured_SKU": true, "whats_new_model": true, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 45.68, + "int8": 36.96, + "fp16": 27.27, "fp32": "", - "bf16": "", - "token_int4": 45.68, - "token_int8": 36.96, - "token_fp16": 27.27 + "bf16": "" } ], "Unit": "FPS", @@ -17218,7 +15859,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17227,10 +15868,7 @@ "int8": 62.13, "fp16": "", "fp32": 5.19, - "bf16": 37.54, - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": 37.54 } ], "Unit": "FPS", @@ -17256,19 +15894,16 @@ "Model": "mistral-7b-v0.1", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 27.42, + "int8": 19.9, + "fp16": 13.72, "fp32": "", - "bf16": "", - "token_int4": 27.42, - "token_int8": 19.9, - "token_fp16": 13.72 + "bf16": "" } ], "Unit": "FPS", @@ -17294,7 +15929,7 @@ "Model": "mobilenet-v2", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17303,10 +15938,7 @@ "int8": 38538.65, "fp16": "", "fp32": 10274.08, - "bf16": 25608.67, - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": 25608.67 } ], "Unit": "FPS", @@ -17331,20 +15963,17 @@ "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", "Model": "phi-3-mini-4k-instruct", "featured_SKU": true, - "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "whats_new_model": true, + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", + "int8": 33.53, + "fp16": 23.1, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": 33.53, - "token_fp16": 23.1 + "bf16": "" } ], "Unit": "FPS", @@ -17370,19 +15999,16 @@ "Model": "qwen2-7b", "featured_SKU": true, "whats_new_model": true, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 30.03, + "int8": 22.14, + "fp16": 13.95, "fp32": "", - "bf16": "", - "token_int4": 30.03, - "token_int8": 22.14, - "token_fp16": 13.95 + "bf16": "" } ], "Unit": "FPS", @@ -17408,7 +16034,7 @@ "Model": "resnet-50", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17417,10 +16043,7 @@ "int8": 19226.96, "fp16": "", "fp32": 1597.37, - "bf16": 7480.12, - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": 7480.12 } ], "Unit": "FPS", @@ -17446,7 +16069,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17455,10 +16078,7 @@ "int8": 434.12, "fp16": "", "fp32": 30.6, - "bf16": 209.11, - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": 209.11 } ], "Unit": "FPS", @@ -17484,7 +16104,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17493,10 +16113,7 @@ "int8": 24134.02, "fp16": "", "fp32": 3392.4, - "bf16": 12168.49, - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": 12168.49 } ], "Unit": "FPS", @@ -17522,7 +16139,7 @@ "Model": "stable-diffusion-v1-5", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17531,10 +16148,7 @@ "int8": "", "fp16": "", "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -17559,8 +16173,8 @@ "Platform": "Intel® Xeon® Platinum 8480+ CPU-only", "Model": "yolo11", "featured_SKU": true, - "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "whats_new_model": "false", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17569,10 +16183,7 @@ "int8": "", "fp16": "", "fp32": 1034.68, - "bf16": 2068.81, - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": 2068.81 } ], "Unit": "FPS", @@ -17598,7 +16209,7 @@ "Model": "yolo_v8n", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17607,10 +16218,7 @@ "int8": 2380.51, "fp16": "", "fp32": 950.6, - "bf16": 2374.89, - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": 2374.89 } ], "Unit": "FPS", @@ -17636,7 +16244,7 @@ "Model": "bert-base-cased", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17645,10 +16253,7 @@ "int8": 4671.04, "fp16": "", "fp32": 560.3, - "bf16": 3211.93, - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": 3211.93 } ], "Unit": "FPS", @@ -17674,7 +16279,7 @@ "Model": "efficientdet-d0", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17683,10 +16288,7 @@ "int8": 1725.13, "fp16": "", "fp32": 1123.04, - "bf16": 1407.69, - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": 1407.69 } ], "Unit": "FPS", @@ -17712,19 +16314,16 @@ "Model": "gemma-2-9b", "featured_SKU": true, "whats_new_model": true, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 25.46, + "int8": 18.96, + "fp16": 12.14, "fp32": "", - "bf16": "", - "token_int4": 25.46, - "token_int8": 18.96, - "token_fp16": 12.14 + "bf16": "" } ], "Unit": "FPS", @@ -17749,20 +16348,17 @@ "Platform": "Intel® Xeon® Platinum 8580 CPU-only", "Model": "glm-4-9b-chat", "featured_SKU": true, - "whats_new_model": true, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "whats_new_model": "false", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 27.1, + "int8": 19.33, + "fp16": 12.69, "fp32": "", - "bf16": "", - "token_int4": 27.1, - "token_int8": 19.33, - "token_fp16": 12.69 + "bf16": "" } ], "Unit": "FPS", @@ -17788,19 +16384,16 @@ "Model": "llama-2-7b-chat", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 31.71, + "int8": 23.05, + "fp16": 16.64, "fp32": "", - "bf16": "", - "token_int4": 31.71, - "token_int8": 23.05, - "token_fp16": 16.64 + "bf16": "" } ], "Unit": "FPS", @@ -17825,20 +16418,17 @@ "Platform": "Intel® Xeon® Platinum 8580 CPU-only", "Model": "llama-3-8b", "featured_SKU": true, - "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "whats_new_model": "false", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 30.06, + "int8": 21.73, + "fp16": 14.93, "fp32": "", - "bf16": "", - "token_int4": 30.06, - "token_int8": 21.73, - "token_fp16": 14.93 + "bf16": "" } ], "Unit": "FPS", @@ -17864,19 +16454,16 @@ "Model": "llama-3.2-3b-instruct", "featured_SKU": true, "whats_new_model": true, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 54.73, + "int8": 42.58, + "fp16": 31.51, "fp32": "", - "bf16": "", - "token_int4": 54.73, - "token_int8": 42.58, - "token_fp16": 31.51 + "bf16": "" } ], "Unit": "FPS", @@ -17902,7 +16489,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17911,10 +16498,7 @@ "int8": 74.86, "fp16": "", "fp32": 6.39, - "bf16": 48.32, - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": 48.32 } ], "Unit": "FPS", @@ -17940,19 +16524,16 @@ "Model": "mistral-7b-v0.1", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 33.27, + "int8": 22.24, + "fp16": 15.74, "fp32": "", - "bf16": "", - "token_int4": 33.27, - "token_int8": 22.24, - "token_fp16": 15.74 + "bf16": "" } ], "Unit": "FPS", @@ -17978,7 +16559,7 @@ "Model": "mobilenet-v2", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -17987,10 +16568,7 @@ "int8": 39894.55, "fp16": "", "fp32": 15839.75, - "bf16": 29419.55, - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": 29419.55 } ], "Unit": "FPS", @@ -18015,20 +16593,17 @@ "Platform": "Intel® Xeon® Platinum 8580 CPU-only", "Model": "phi-3-mini-4k-instruct", "featured_SKU": true, - "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "whats_new_model": true, + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { "int4": "", - "int8": "", - "fp16": "", + "int8": 40.45, + "fp16": 26.95, "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": 40.45, - "token_fp16": 26.95 + "bf16": "" } ], "Unit": "FPS", @@ -18037,9 +16612,9 @@ "latency": { "Precisions": [ { - "int4": 21.43, + "int4": "", "int8": 24.72, - "fp16": "", + "fp16": 37.1, "fp32": "", "bf16": "" } @@ -18054,19 +16629,16 @@ "Model": "qwen2-7b", "featured_SKU": true, "whats_new_model": true, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", - "fp16": "", + "int4": 35.48, + "int8": 25.7, + "fp16": 16.1, "fp32": "", - "bf16": "", - "token_int4": 35.48, - "token_int8": 25.7, - "token_fp16": 16.1 + "bf16": "" } ], "Unit": "FPS", @@ -18092,7 +16664,7 @@ "Model": "resnet-50", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18101,10 +16673,7 @@ "int8": 21612.82, "fp16": "", "fp32": 2002.36, - "bf16": 13669.05, - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": 13669.05 } ], "Unit": "FPS", @@ -18130,7 +16699,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18139,10 +16708,7 @@ "int8": 513.09, "fp16": "", "fp32": 35.2, - "bf16": 275.94, - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": 275.94 } ], "Unit": "FPS", @@ -18168,7 +16734,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18177,10 +16743,7 @@ "int8": 26748.89, "fp16": "", "fp32": 4718.18, - "bf16": 16684.87, - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": 16684.87 } ], "Unit": "FPS", @@ -18206,7 +16769,7 @@ "Model": "stable-diffusion-v1-5", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18215,10 +16778,7 @@ "int8": "", "fp16": "", "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "FPS", @@ -18243,8 +16803,8 @@ "Platform": "Intel® Xeon® Platinum 8580 CPU-only", "Model": "yolo11", "featured_SKU": true, - "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "whats_new_model": "false", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18253,10 +16813,7 @@ "int8": "", "fp16": "", "fp32": 1455.5, - "bf16": 2962.49, - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": 2962.49 } ], "Unit": "FPS", @@ -18282,7 +16839,7 @@ "Model": "yolo_v8n", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18291,10 +16848,7 @@ "int8": 3043.23, "fp16": "", "fp32": 1258.2, - "bf16": 3444.22, - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": 3444.22 } ], "Unit": "FPS", @@ -18320,7 +16874,7 @@ "Model": "bert-base-cased", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18329,10 +16883,7 @@ "int8": 8897.30, "fp16": "", "fp32": 1217.03, - "bf16": 6414.49, - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": 6414.49 } ], "Unit": "FPS", @@ -18358,7 +16909,7 @@ "Model": "efficientdet-d0", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18367,10 +16918,7 @@ "int8": 3384.23, "fp16": "", "fp32": 2295.4, - "bf16": 2872.84, - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": 2872.84 } ], "Unit": "FPS", @@ -18396,7 +16944,7 @@ "Model": "mask_rcnn_resnet50_atrous_coco", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18405,10 +16953,7 @@ "int8": 149.52, "fp16": "", "fp32": 11.97, - "bf16": 91.85, - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": 91.85 } ], "Unit": "FPS", @@ -18434,7 +16979,7 @@ "Model": "mobilenet-v2", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18443,10 +16988,7 @@ "int8": 32737.09, "fp16": "", "fp32": 25621.92, - "bf16": 26297.21, - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": 26297.21 } ], "Unit": "FPS", @@ -18472,7 +17014,7 @@ "Model": "resnet-50", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18481,10 +17023,7 @@ "int8": 27670.82, "fp16": "", "fp32": 4254.94, - "bf16": 22432.74, - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": 22432.74 } ], "Unit": "FPS", @@ -18510,7 +17049,7 @@ "Model": "ssd-resnet34-1200", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18519,10 +17058,7 @@ "int8": 1009.62, "fp16": "", "fp32": 77.99, - "bf16": 532.90, - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": 532.90 } ], "Unit": "FPS", @@ -18548,7 +17084,7 @@ "Model": "ssd_mobilenet_v1_coco", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18557,10 +17093,7 @@ "int8": 29674.40, "fp16": "", "fp32": 9800.83, - "bf16": 19479.18, - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": 19479.18 } ], "Unit": "FPS", @@ -18586,7 +17119,7 @@ "Model": "yolo_v8n", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18595,10 +17128,7 @@ "int8": 5590.87, "fp16": "", "fp32": 2699.0, - "bf16": 6003.66, - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": 6003.66 } ], "Unit": "FPS", @@ -18624,19 +17154,16 @@ "Model": "gemma-2-9b", "featured_SKU": true, "whats_new_model": true, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", + "int4": 136.4, "int8": "", - "fp16": "", + "fp16": 53.6, "fp32": "", - "bf16": "", - "token_int4": 136.4, - "token_int8": "", - "token_fp16": 53.6 + "bf16": "" } ], "Unit": "Tokens/sec", @@ -18662,19 +17189,16 @@ "Model": "glm-4-9b-chat", "featured_SKU": true, "whats_new_model": true, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", + "int4": 116.5, "int8": "", - "fp16": "", + "fp16": 51.9, "fp32": "", - "bf16": "", - "token_int4": 116.5, - "token_int8": "", - "token_fp16": 51.9 + "bf16": "" } ], "Unit": "Tokens/sec", @@ -18700,19 +17224,16 @@ "Model": "llama-2-7b-chat", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", + "int4": 139.5, "int8": "", - "fp16": "", + "fp16": 132, "fp32": "", - "bf16": "", - "token_int4": 139.5, - "token_int8": "", - "token_fp16": 132 + "bf16": "" } ], "Unit": "Tokens/sec", @@ -18738,19 +17259,16 @@ "Model": "llama-3.2-3b-instruct", "featured_SKU": true, "whats_new_model": true, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", - "int8": "", + "int4": 272.7, + "int8": 65, "fp16": "", "fp32": "", - "bf16": "", - "token_int4": 272.7, - "token_int8": 65, - "token_fp16": "" + "bf16": "" } ], "Unit": "Tokens/sec", @@ -18776,19 +17294,16 @@ "Model": "llama-3-8b", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", + "int4": 148.2, "int8": "", - "fp16": "", + "fp16": 57.2, "fp32": "", - "bf16": "", - "token_int4": 148.2, - "token_int8": "", - "token_fp16": 57.2 + "bf16": "" } ], "Unit": "Tokens/sec", @@ -18814,19 +17329,16 @@ "Model": "mistral-7b-v0.1", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", + "int4": 126.4, "int8": "", - "fp16": "", + "fp16": 61.4, "fp32": "", - "bf16": "", - "token_int4": 126.4, - "token_int8": "", - "token_fp16": 61.4 + "bf16": "" } ], "Unit": "Tokens/sec", @@ -18852,19 +17364,16 @@ "Model": "phi-3-mini-4k-instruct", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", + "int4": 176.6, "int8": "", - "fp16": "", + "fp16": 111.9, "fp32": "", - "bf16": "", - "token_int4": 176.6, - "token_int8": "", - "token_fp16": 111.9 + "bf16": "" } ], "Unit": "Tokens/sec", @@ -18890,19 +17399,16 @@ "Model": "qwen2-7b", "featured_SKU": true, "whats_new_model": true, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ { - "int4": "", + "int4": 164.4, "int8": "", - "fp16": "", + "fp16": 62.2, "fp32": "", - "bf16": "", - "token_int4": 164.4, - "token_int8": "", - "token_fp16": 62.2 + "bf16": "" } ], "Unit": "Tokens/sec", @@ -18928,7 +17434,7 @@ "Model": "stable-diffusion-v1-5", "featured_SKU": true, "whats_new_model": false, - "PlatformType": "Server Platforms (Intel® Xeon®)", + "PlatformType": "Intel® Xeon®, CPU-only", "Parameters": { "throughput": { "Precisions": [ @@ -18937,10 +17443,7 @@ "int8": "", "fp16": "", "fp32": "", - "bf16": "", - "token_int4": "", - "token_int8": "", - "token_fp16": "" + "bf16": "" } ], "Unit": "n/a", diff --git a/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ovms-genai.json b/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ovms-genai.json index f96fb11e6b029d..0de8f188e7de34 100644 --- a/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ovms-genai.json +++ b/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ovms-genai.json @@ -1,45 +1,330 @@ [ + { + "Platform": "Intel® Xeon® Platinum 8380", + "Model": "meta-llama/Llama-2-7b-chat-hf", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", + "Parameters": { + "Ovms": { + "Precisions": [ + { + "Throughput": { + "0.2": 94.97, + "0.4": 187.12, + "0.6": 271.85, + "0.8": 290.81, + "1.0": 291.39, + "2.0": 291.45, + "inf": 291.59 + }, + "Latency": { + "0.2": 74.35, + "0.4": 122.25, + "0.6": 467.49, + "0.8": 749.39, + "1.0": 771.39, + "2.0": 773.31, + "inf": 783.63 + } + } + ] + }, + "Vllm": { + "Precisions": [ + { + "Throughput": { + "0.2": 94.83, + "0.4": 187.83, + "0.6": 272.32, + "0.8": 284.07, + "1.0": 291.88, + "2.0": 291.91, + "inf": 288.62 + }, + "Latency": { + "0.2": 82.31, + "0.4": 134.38, + "0.6": 495.99, + "0.8": 794.41, + "1.0": 798.39, + "2.0": 800.33, + "inf": 809.56 + } + } + ] + } + } + }, + { + "Platform": "Intel® Xeon® Platinum 8480+", + "Model": "meta-llama/Llama-2-7b-chat-hf", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", + "Parameters": { + "Ovms": { + "Precisions": [ + { + "Throughput": { + "0.2": 95.15, + "0.4": 188.31, + "0.6": 279.3, + "0.8": 366.78, + "1.0": 454.27, + "2.0": 788.9, + "inf": 825.97 + }, + "Latency": { + "0.2": 60.88, + "0.4": 71.96, + "0.6": 83.45, + "0.8": 103.77, + "1.0": 128.12, + "2.0": 237.62, + "inf": 253.59 + } + } + ] + }, + "Vllm": { + "Precisions": [ + { + "Throughput": { + "0.2": 95.06, + "0.4": 188.47, + "0.6": 280.54, + "0.8": 367.47, + "1.0": 450.81, + "2.0": 774.57, + "inf": 793.78 + }, + "Latency": { + "0.2": 63.84, + "0.4": 76.22, + "0.6": 87.21, + "0.8": 104.75, + "1.0": 136.77, + "2.0": 259.2, + "inf": 273.58 + } + } + ] + } + } + }, { "Platform": "Intel® Xeon® Platinum 8580", - "Model": "mistralai/Mistral-7B-v0.1", - "PlatformType": "None", + "Model": "meta-llama/Llama-2-7b-chat-hf", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { + "Ovms": { + "Precisions": [ + { + "Throughput": { + "0.2": 95.29, + "0.4": 188.33, + "0.6": 280.09, + "0.8": 367.29, + "1.0": 453.21, + "2.0": 780.05, + "inf": 751.34 + }, + "Latency": { + "0.2": 52.44, + "0.4": 70.06, + "0.6": 84.54, + "0.8": 108.91, + "1.0": 136.45, + "2.0": 253.55, + "inf": 281.85 + } + } + ] + }, "Vllm": { "Precisions": [ { "Throughput": { - "0.2": "350.06", - "0.6": "486.89", - "0.8": "575.92", - "2.0": "778.07" + "0.2": 95.0, + "0.4": 188.26, + "0.6": 279.78, + "0.8": 366.69, + "1.0": 450.26, + "2.0": 770.74, + "inf": 794.39 + }, + "Latency": { + "0.2": 58.07, + "0.4": 77.65, + "0.6": 91.14, + "0.8": 113.61, + "1.0": 144.21, + "2.0": 269.13, + "inf": 273.27 } - }, + } + ] + } + } + }, + { + "Platform": "Intel® Xeon® Platinum 8380", + "Model": "meta-llama/Meta-Llama-3-8B-Instruct", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Server Platforms (Intel® Xeon®)", + "Parameters": { + "Ovms": { + "Precisions": [ { + "Throughput": { + "0.2": 82.46, + "0.4": 162.73, + "0.6": 240.08, + "0.8": 273.75, + "1.0": 275.85, + "2.0": 276.3, + "inf": 275.15 + }, "Latency": { - "0.2": "60.93", - "0.6": "91.63", - "0.8": "113.61", - "2.0": "240.25" + "0.2": 76.49, + "0.4": 122.1, + "0.6": 318.14, + "0.8": 785.8, + "1.0": 805.58, + "2.0": 809.37, + "inf": 816.2 } } ] }, + "Vllm": { + "Precisions": [ + { + "Throughput": { + "0.2": 82.32, + "0.4": 162.98, + "0.6": 239.28, + "2.0": 270.37 + }, + "Latency": { + "0.2": 87.92, + "0.4": 142.3, + "0.6": 343.36, + "2.0": 873.0 + } + } + ] + } + } + }, + { + "Platform": "Intel® Xeon® Platinum 8480+", + "Model": "meta-llama/Meta-Llama-3-8B-Instruct", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Server Platforms (Intel® Xeon®)", + "Parameters": { + "Ovms": { + "Precisions": [ + { + "Throughput": { + "0.2": 82.61, + "0.4": 164.44, + "0.6": 244.92, + "0.8": 323.34, + "1.0": 400.78, + "2.0": 731.9, + "inf": 848.45 + }, + "Latency": { + "0.2": 60.77, + "0.4": 69.1, + "0.6": 74.36, + "0.8": 81.41, + "1.0": 100.17, + "2.0": 206.5, + "inf": 246.56 + } + } + ] + }, + "Vllm": { + "Precisions": [ + { + "Throughput": { + "0.2": 82.54, + "0.4": 163.66, + "0.6": 243.88, + "0.8": 322.75, + "1.0": 400.46, + "2.0": 727.1 + }, + "Latency": { + "0.2": 65.37, + "0.4": 75.87, + "0.6": 81.14, + "0.8": 93.91, + "1.0": 107.13, + "2.0": 229.57 + } + } + ] + } + } + }, + { + "Platform": "Intel® Xeon® Platinum 8580", + "Model": "meta-llama/Meta-Llama-3-8B-Instruct", + "featured_SKU": true, + "whats_new_model": true, + "PlatformType": "Server Platforms (Intel® Xeon®)", + "Parameters": { "Ovms": { "Precisions": [ { "Throughput": { - "0.2": "90.98", - "0.6": "266.24", - "0.8": "351.63", - "2.0": "195.16" + "0.2": 82.55, + "0.4": 164.52, + "0.6": 243.96, + "0.8": 323.07, + "1.0": 399.68, + "2.0": 727.18, + "inf": 856.72 + }, + "Latency": { + "0.2": 54.57, + "0.4": 69.17, + "0.6": 80.32, + "0.8": 92.94, + "1.0": 111.06, + "2.0": 215.46, + "inf": 245.72 } - }, + } + ] + }, + "Vllm": { + "Precisions": [ { + "Throughput": { + "0.2": 82.64, + "0.6": 243.81, + "0.8": 321.8, + "1.0": 398.78, + "2.0": 722.48, + "inf": 792.34 + }, "Latency": { - "0.2": "54.9", - "0.6": "78.78", - "0.8": "95.78", - "2.0": "352.23" + "0.2": 61.49, + "0.6": 90.54, + "0.8": 106.25, + "1.0": 123.6, + "2.0": 245.91, + "inf": 279.21 } } ] @@ -47,46 +332,168 @@ } }, { - "Platform": "Intel® Xeon® Platinum 8530", + "Platform": "Intel® Xeon® Platinum 8380", "Model": "mistralai/Mistral-7B-v0.1", - "PlatformType": "None", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", "Parameters": { + "Ovms": { + "Precisions": [ + { + "Throughput": { + "0.2": 91.74, + "0.4": 180.4, + "0.6": 262.97, + "0.8": 287.36, + "1.0": 289.08, + "2.0": 289.06, + "inf": 290.69 + }, + "Latency": { + "0.2": 74.84, + "0.4": 115.4, + "0.6": 345.64, + "0.8": 757.42, + "1.0": 776.6, + "2.0": 778.29, + "inf": 784.42 + } + } + ] + }, "Vllm": { "Precisions": [ { "Throughput": { - "0.2": "350.06", - "0.6": "486.89", - "0.8": "575.92", - "2.0": "778.07" + "0.2": 97.21, + "0.4": 192.46, + "0.6": 265.82, + "0.8": 273.24, + "1.0": 272.65, + "inf": 274.0 + }, + "Latency": { + "0.2": 166.77, + "0.4": 161.76, + "0.6": 666.89, + "0.8": 802.15, + "1.0": 810.26, + "inf": 807.71 } - }, + } + ] + } + } + }, + { + "Platform": "Intel® Xeon® Platinum 8480+", + "Model": "mistralai/Mistral-7B-v0.1", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", + "Parameters": { + "Ovms": { + "Precisions": [ { + "Throughput": { + "0.2": 90.95, + "0.4": 181.06, + "0.6": 267.29, + "0.8": 351.62, + "1.0": 431.45, + "2.0": 751.85, + "inf": 596.0 + }, "Latency": { - "0.2": "60.93", - "0.6": "91.63", - "0.8": "113.61", - "2.0": "240.25" + "0.2": 59.95, + "0.4": 63.41, + "0.6": 73.42, + "0.8": 85.99, + "1.0": 98.67, + "2.0": 205.2, + "inf": 205.97 } } ] }, + "Vllm": { + "Precisions": [ + { + "Throughput": { + "0.2": 98.18, + "0.4": 194.35, + "0.6": 287.28, + "0.8": 376.31, + "1.0": 460.32, + "2.0": 771.81, + "inf": 789.38 + }, + "Latency": { + "0.2": 64.88, + "0.4": 73.3, + "0.6": 84.37, + "0.8": 100.8, + "1.0": 133.98, + "2.0": 240.99, + "inf": 251.55 + } + } + ] + } + } + }, + { + "Platform": "Intel® Xeon® Platinum 8580", + "Model": "mistralai/Mistral-7B-v0.1", + "featured_SKU": true, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", + "Parameters": { "Ovms": { "Precisions": [ { "Throughput": { - "0.2": "90.98", - "0.6": "266.24", - "0.8": "351.63", - "2.0": "195.16" + "0.2": 91.2, + "0.4": 180.14, + "0.6": 267.75, + "0.8": 351.12, + "1.0": 428.31, + "2.0": 744.99, + "inf": 852.05 + }, + "Latency": { + "0.2": 54.31, + "0.4": 67.14, + "0.6": 77.59, + "0.8": 92.17, + "1.0": 112.75, + "2.0": 225.48, + "inf": 241.49 } - }, + } + ] + }, + "Vllm": { + "Precisions": [ { + "Throughput": { + "0.2": 98.1, + "0.4": 194.47, + "0.6": 286.97, + "0.8": 375.84, + "1.0": 460.21, + "2.0": 764.54, + "inf": 787.97 + }, "Latency": { - "0.2": "54.9", - "0.6": "78.78", - "0.8": "95.78", - "2.0": "352.23" + "0.2": 62.26, + "0.4": 78.08, + "0.6": 91.61, + "0.8": 116.71, + "1.0": 141.76, + "2.0": 250.38, + "inf": 254.25 } } ] diff --git a/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ovms.json b/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ovms.json index 18a36073d582f5..f601a8120117d6 100644 --- a/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ovms.json +++ b/docs/sphinx_setup/_static/benchmarks_files/data/graph-data-ovms.json @@ -1,1047 +1,1283 @@ [ - { - "Platform": "Intel® Xeon® Gold 6238M", - "Model": "bert-base-cased", - "PlatformType": "Server Platforms (Intel® Xeon®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 159.534, - "fp32_ovms": 157.334, - "int8_ov": 432.339, - "int8_ovms": 420.793 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Xeon® Gold 6238M", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "PlatformType": "Server Platforms (Intel® Xeon®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 13.125, - "fp32_ovms": 13.254, - "int8_ov": 38.151, - "int8_ovms": 37.623 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Xeon® Gold 6238M", - "Model": "efficientdet-d0", - "PlatformType": "Server Platforms (Intel® Xeon®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 247.445, - "fp32_ovms": 253.09, - "int8_ov": 413.083, - "int8_ovms": 377.844 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Xeon® Gold 6238M", - "Model": "mask_rcnn_resnet50_atrous_coco", - "PlatformType": "Server Platforms (Intel® Xeon®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 1.638, - "fp32_ovms": 1.714, - "int8_ov": 6.202, - "int8_ovms": 6.126 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Xeon® Gold 6238M", - "Model": "mobilenet-v2", - "PlatformType": "Server Platforms (Intel® Xeon®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 3333.399, - "fp32_ovms": 2905.171, - "int8_ov": 10422.241, - "int8_ovms": 7461.99 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Xeon® Gold 6238M", - "Model": "resnet-50", - "PlatformType": "Server Platforms (Intel® Xeon®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 575.208, - "fp32_ovms": 569.925, - "int8_ov": 2199.072, - "int8_ovms": 2064.581 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Xeon® Gold 6238M", - "Model": "ssd-resnet34-1200", - "PlatformType": "Server Platforms (Intel® Xeon®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 10.598, - "fp32_ovms": 10.472, - "int8_ov": 40.683, - "int8_ovms": 38.737 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Xeon® Gold 6238M", - "Model": "ssd_mobilenet_v1_coco", - "PlatformType": "Server Platforms (Intel® Xeon®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 1219.441, - "fp32_ovms": 1201.096, - "int8_ov": 4400.471, - "int8_ovms": 4270.702 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Xeon® Gold 6238M", - "Model": "unet-camvid-onnx-0001", - "PlatformType": "Server Platforms (Intel® Xeon®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 15.924, - "fp32_ovms": 15.763, - "int8_ov": 67.731, - "int8_ovms": 64.658 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Xeon® Gold 6238M", - "Model": "yolo_v5m", - "PlatformType": "Server Platforms (Intel® Xeon®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 74.189, - "fp32_ovms": 68.788, - "int8_ov": 247.757, - "int8_ovms": 180.302 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Xeon® Gold 6238M", - "Model": "yolo_v8n", - "PlatformType": "Server Platforms (Intel® Xeon®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 332.326, - "fp32_ovms": 278.054, - "int8_ov": 740.985, - "int8_ovms": 609.062 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8260M", - "Model": "bert-base-cased", - "PlatformType": "Server Platforms (Intel® Xeon®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 182.025, - "fp32_ovms": 180.764, - "int8_ov": 485.82, - "int8_ovms": 472.842 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8260M", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "PlatformType": "Server Platforms (Intel® Xeon®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 14.625, - "fp32_ovms": 15.132, - "int8_ov": 42.906, - "int8_ovms": 42.406 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8260M", - "Model": "efficientdet-d0", - "PlatformType": "Server Platforms (Intel® Xeon®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 288.531, - "fp32_ovms": 278.548, - "int8_ov": 483.438, - "int8_ovms": 443.032 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8260M", - "Model": "mask_rcnn_resnet50_atrous_coco", - "PlatformType": "Server Platforms (Intel® Xeon®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 1.872, - "fp32_ovms": 1.95, - "int8_ov": 6.856, - "int8_ovms": 6.763 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8260M", - "Model": "mobilenet-v2", - "PlatformType": "Server Platforms (Intel® Xeon®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 3909.405, - "fp32_ovms": 3327.621, - "int8_ov": 12375.018, - "int8_ovms": 7554.235 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8260M", - "Model": "resnet-50", - "PlatformType": "Server Platforms (Intel® Xeon®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 634.732, - "fp32_ovms": 634.102, - "int8_ov": 2481.256, - "int8_ovms": 2349.872 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8260M", - "Model": "ssd-resnet34-1200", - "PlatformType": "Server Platforms (Intel® Xeon®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 12.166, - "fp32_ovms": 12.027, - "int8_ov": 47.295, - "int8_ovms": 44.525 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8260M", - "Model": "ssd_mobilenet_v1_coco", - "PlatformType": "Server Platforms (Intel® Xeon®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 1384.145, - "fp32_ovms": 1356.126, - "int8_ov": 5037.197, - "int8_ovms": 4834.045 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8260M", - "Model": "unet-camvid-onnx-0001", - "PlatformType": "Server Platforms (Intel® Xeon®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 18.26, - "fp32_ovms": 18.052, - "int8_ov": 77.933, - "int8_ovms": 73.527 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8260M", - "Model": "yolo_v5m", - "PlatformType": "Server Platforms (Intel® Xeon®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 85.149, - "fp32_ovms": 78.205, - "int8_ov": 281.889, - "int8_ovms": 204.353 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Xeon® Platinum 8260M", - "Model": "yolo_v8n", - "PlatformType": "Server Platforms (Intel® Xeon®)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 376.079, - "fp32_ovms": 312.181, - "int8_ov": 801.556, - "int8_ovms": 678.929 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-11700K", - "Model": "bert-base-cased", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 35.915, - "fp32_ovms": 34.381, - "int8_ov": 101.976, - "int8_ovms": 99.024 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-11700K", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 3.232, - "fp32_ovms": 3.266, - "int8_ov": 10.132, - "int8_ovms": 10.133 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-11700K", - "Model": "efficientdet-d0", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 51.747, - "fp32_ovms": 48.906, - "int8_ov": 142.489, - "int8_ovms": 124.167 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-11700K", - "Model": "mask_rcnn_resnet50_atrous_coco", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 0.352, - "fp32_ovms": 0.364, - "int8_ov": 1.322, - "int8_ovms": 1.336 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-11700K", - "Model": "mobilenet-v2", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 795.18, - "fp32_ovms": 664.842, - "int8_ov": 2721.454, - "int8_ovms": 2063.761 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-11700K", - "Model": "resnet-50", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 114.859, - "fp32_ovms": 110.835, - "int8_ov": 467.591, - "int8_ovms": 445.408 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-11700K", - "Model": "ssd-resnet34-1200", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 2.053, - "fp32_ovms": 2.074, - "int8_ov": 8.023, - "int8_ovms": 7.987 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-11700K", - "Model": "ssd_mobilenet_v1_coco", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 260.104, - "fp32_ovms": 250.094, - "int8_ov": 991.064, - "int8_ovms": 930.128 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-11700K", - "Model": "unet-camvid-onnx-0001", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 3.273, - "fp32_ovms": 3.3, - "int8_ov": 12.884, - "int8_ovms": 12.727 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-11700K", - "Model": "yolo_v5m", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 14.714, - "fp32_ovms": 14.243, - "int8_ov": 55.058, - "int8_ovms": 47.548 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i7-11700K", - "Model": "yolo_v8n", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 71.446, - "fp32_ovms": 64.775, - "int8_ov": 200.864, - "int8_ovms": 144.792 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i9-11900K", - "Model": "bert-base-cased", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 36.227, - "fp32_ovms": 35.646, - "int8_ov": 101.562, - "int8_ovms": 100.382 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i9-11900K", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 3.23, - "fp32_ovms": 3.254, - "int8_ov": 10.05, - "int8_ovms": 10.092 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i9-11900K", - "Model": "efficientdet-d0", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 59.759, - "fp32_ovms": 55.851, - "int8_ov": 149.505, - "int8_ovms": 131.453 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i9-11900K", - "Model": "mask_rcnn_resnet50_atrous_coco", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 0.368, - "fp32_ovms": 0.394, - "int8_ov": 1.308, - "int8_ovms": 1.338 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i9-11900K", - "Model": "mobilenet-v2", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 865.806, - "fp32_ovms": 734.822, - "int8_ov": 2743.201, - "int8_ovms": 2163.412 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i9-11900K", - "Model": "resnet-50", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 116.784, - "fp32_ovms": 113.046, - "int8_ov": 457.358, - "int8_ovms": 440.924 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i9-11900K", - "Model": "ssd-resnet34-1200", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 2.006, - "fp32_ovms": 2.031, - "int8_ov": 7.817, - "int8_ovms": 7.75 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i9-11900K", - "Model": "ssd_mobilenet_v1_coco", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 274.42, - "fp32_ovms": 264.153, - "int8_ov": 997.987, - "int8_ovms": 915.681 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i9-11900K", - "Model": "unet-camvid-onnx-0001", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 3.246, - "fp32_ovms": 3.272, - "int8_ov": 12.668, - "int8_ovms": 12.585 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i9-11900K", - "Model": "yolo_v5m", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 14.985, - "fp32_ovms": 14.514, - "int8_ov": 54.937, - "int8_ovms": 47.767 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i9-11900K", - "Model": "yolo_v8n", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 74.1, - "fp32_ovms": 67.472, - "int8_ov": 203.493, - "int8_ovms": 151.175 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i3-10100", - "Model": "bert-base-cased", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 17.054, - "fp32_ovms": 17.124, - "int8_ov": 26.043, - "int8_ovms": 25.872 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i3-10100", - "Model": "bert-large-uncased-whole-word-masking-squad-0001", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 1.434, - "fp32_ovms": 1.456, - "int8_ov": 2.421, - "int8_ovms": 2.450 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i3-10100", - "Model": "efficientdet-d0", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 31.321, - "fp32_ovms": 30.316, - "int8_ov": 50.629, - "int8_ovms": 47.377 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i3-10100", - "Model": "mask_rcnn_resnet50_atrous_coco", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 0.151, - "fp32_ovms": 0.182, - "int8_ov": 0.361, - "int8_ovms": 0.389 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i3-10100", - "Model": "mobilenet-v2", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 442.763, - "fp32_ovms": 380.661, - "int8_ov": 724.232, - "int8_ovms": 617.393 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i3-10100", - "Model": "resnet-50", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 57.978, - "fp32_ovms": 57.038, - "int8_ov": 118.213, - "int8_ovms": 113.691 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i3-10100", - "Model": "ssd-resnet34-1200", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 1.0, - "fp32_ovms": 1.031, - "int8_ov": 1.937, - "int8_ovms": 1.954 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i3-10100", - "Model": "ssd_mobilenet_v1_coco", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 133.421, - "fp32_ovms": 129.949, - "int8_ov": 267.141, - "int8_ovms": 256.821 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i3-10100", - "Model": "unet-camvid-onnx-0001", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 1.515, - "fp32_ovms": 1.534, - "int8_ov": 2.96, - "int8_ovms": 2.973 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i3-10100", - "Model": "yolo_v5m", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 7.691, - "fp32_ovms": 7.511, - "int8_ov": 14.919, - "int8_ovms": 13.832 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - }, - { - "Platform": "Intel® Core™ i3-10100", - "Model": "yolo_v8n", - "PlatformType": "Client Platforms (Intel® Core™)", - "Parameters": { - "throughput": { - "Precisions": [ - { - "fp32_ov": 38.482, - "fp32_ovms": 34.513, - "int8_ov": 68.126, - "int8_ovms": 55.698 - } - ], - "Unit": "FPS", - "UnitDesc": "higher is better" - } - } - } + { + "Platform": "Intel® Xeon® Gold 6238M", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 161.041, + "fp32_ovms": 157.547, + "int8_ov": 435.257, + "int8_ovms": 422.689 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Xeon® Gold 6238M", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 259.576, + "fp32_ovms": 256.524, + "int8_ov": 412.419, + "int8_ovms": 376.69 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Xeon® Gold 6238M", + "Model": "manual_yolo11", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Server Platforms (Intel® Xeon®)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 385.381, + "fp32_ovms": 312.784, + "int8_ov": "", + "int8_ovms": "" + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Xeon® Gold 6238M", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 1.64, + "fp32_ovms": 1.718, + "int8_ov": 6.426, + "int8_ovms": 6.258 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Xeon® Gold 6238M", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 3349.207, + "fp32_ovms": 2904.878, + "int8_ov": 10365.087, + "int8_ovms": 7521.115 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Xeon® Gold 6238M", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 580.494, + "fp32_ovms": 572.921, + "int8_ov": 2196.814, + "int8_ovms": 2072.444 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Xeon® Gold 6238M", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 10.627, + "fp32_ovms": 10.524, + "int8_ov": 40.619, + "int8_ovms": 38.733 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Xeon® Gold 6238M", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 1234.49, + "fp32_ovms": 1203.314, + "int8_ov": 4445.793, + "int8_ovms": 4261.084 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Xeon® Gold 6238M", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 337.397, + "fp32_ovms": 279.585, + "int8_ov": 758.758, + "int8_ovms": 641.433 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Xeon® Platinum 8260M", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 182.454, + "fp32_ovms": 181.015, + "int8_ov": 487.412, + "int8_ovms": 475.32 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Xeon® Platinum 8260M", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 291.999, + "fp32_ovms": 289.402, + "int8_ov": 485.657, + "int8_ovms": 442.145 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Xeon® Platinum 8260M", + "Model": "manual_yolo11", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Server Platforms (Intel® Xeon®)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 437.295, + "fp32_ovms": 354.521, + "int8_ov": "", + "int8_ovms": "" + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Xeon® Platinum 8260M", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 1.889, + "fp32_ovms": 1.961, + "int8_ov": 7.085, + "int8_ovms": 6.985 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Xeon® Platinum 8260M", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 3923.365, + "fp32_ovms": 3332.521, + "int8_ov": 12328.807, + "int8_ovms": 7562.762 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Xeon® Platinum 8260M", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 645.638, + "fp32_ovms": 639.958, + "int8_ov": 2493.033, + "int8_ovms": 2349.919 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Xeon® Platinum 8260M", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 12.196, + "fp32_ovms": 12.091, + "int8_ov": 47.197, + "int8_ovms": 44.379 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Xeon® Platinum 8260M", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 1385.809, + "fp32_ovms": 1374.891, + "int8_ov": 5079.624, + "int8_ovms": 4836.539 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Xeon® Platinum 8260M", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Server Platforms (Intel® Xeon®)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 383.81, + "fp32_ovms": 315.245, + "int8_ov": 858.66, + "int8_ovms": 704.713 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i7-11700K", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 34.685, + "fp32_ovms": 32.405, + "int8_ov": 100.893, + "int8_ovms": 94.564 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i7-11700K", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 51.125, + "fp32_ovms": 46.351, + "int8_ov": 141.548, + "int8_ovms": 115.788 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i7-11700K", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 0.331, + "fp32_ovms": 0.336, + "int8_ov": 1.331, + "int8_ovms": 1.354 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i7-11700K", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 774.104, + "fp32_ovms": 628.503, + "int8_ov": 2723.303, + "int8_ovms": 1832.886 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i7-11700K", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 113.363, + "fp32_ovms": 106.029, + "int8_ov": 466.473, + "int8_ovms": 433.532 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i7-11700K", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 2.047, + "fp32_ovms": 2.047, + "int8_ov": 8.016, + "int8_ovms": 7.886 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i7-11700K", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 259.504, + "fp32_ovms": 236.341, + "int8_ov": 995.124, + "int8_ovms": 869.518 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i7-11700K", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 71.24, + "fp32_ovms": 62.319, + "int8_ov": 199.772, + "int8_ovms": 133.145 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i9-11900K", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 36.251, + "fp32_ovms": 35.465, + "int8_ov": 101.305, + "int8_ovms": 99.151 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i9-11900K", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 59.247, + "fp32_ovms": 55.459, + "int8_ov": 148.119, + "int8_ovms": 130.171 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i9-11900K", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 0.37, + "fp32_ovms": 0.388, + "int8_ov": 1.321, + "int8_ovms": 1.332 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i9-11900K", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 855.526, + "fp32_ovms": 713.553, + "int8_ov": 2745.282, + "int8_ovms": 2129.129 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i9-11900K", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 115.109, + "fp32_ovms": 112.189, + "int8_ov": 455.027, + "int8_ovms": 437.03 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i9-11900K", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 2.004, + "fp32_ovms": 2.022, + "int8_ov": 7.796, + "int8_ovms": 7.729 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i9-11900K", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 274.523, + "fp32_ovms": 260.272, + "int8_ov": 966.639, + "int8_ovms": 893.165 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i9-11900K", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 74.006, + "fp32_ovms": 67.143, + "int8_ov": 204.296, + "int8_ovms": 151.136 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i3-10100", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 17.146, + "fp32_ovms": 17.085, + "int8_ov": 26.112, + "int8_ovms": 25.962 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i3-10100", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 30.601, + "fp32_ovms": 29.76, + "int8_ov": 49.646, + "int8_ovms": 47.222 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i3-10100", + "Model": "manual_yolo11", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 43.751, + "fp32_ovms": 38.752, + "int8_ov": "", + "int8_ovms": "" + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i3-10100", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 0.148, + "fp32_ovms": 0.18, + "int8_ov": 0.36, + "int8_ovms": 0.39 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i3-10100", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 440.453, + "fp32_ovms": 380.439, + "int8_ov": 714.915, + "int8_ovms": 611.391 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i3-10100", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 57.896, + "fp32_ovms": 56.88, + "int8_ov": 117.702, + "int8_ovms": 113.447 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i3-10100", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 0.996, + "fp32_ovms": 1.033, + "int8_ov": 1.935, + "int8_ovms": 1.946 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i3-10100", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 132.73, + "fp32_ovms": 128.89, + "int8_ov": 266.502, + "int8_ovms": 256.113 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i3-10100", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 38.386, + "fp32_ovms": 34.599, + "int8_ov": 68.072, + "int8_ovms": 55.668 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i7-11700K", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 34.685, + "fp32_ovms": 33.575, + "int8_ov": 100.893, + "int8_ovms": 96.251 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i7-11700K", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 51.125, + "fp32_ovms": 47.06, + "int8_ov": 141.548, + "int8_ovms": 117.642 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i7-11700K", + "Model": "manual_yolo11", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 80.399, + "fp32_ovms": 68.631, + "int8_ov": "", + "int8_ovms": "" + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i7-11700K", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 0.331, + "fp32_ovms": 0.344, + "int8_ov": 1.331, + "int8_ovms": 1.417 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i7-11700K", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 774.104, + "fp32_ovms": 628.386, + "int8_ov": 2723.303, + "int8_ovms": 1905.703 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i7-11700K", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 113.363, + "fp32_ovms": 106.07, + "int8_ov": 466.473, + "int8_ovms": 433.345 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i7-11700K", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 2.047, + "fp32_ovms": 2.055, + "int8_ov": 8.016, + "int8_ovms": 7.884 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i7-11700K", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 259.504, + "fp32_ovms": 238.91, + "int8_ov": 995.124, + "int8_ovms": 880.377 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i7-11700K", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 71.24, + "fp32_ovms": 62.386, + "int8_ov": 199.772, + "int8_ovms": 139.345 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i9-11900K", + "Model": "bert-base-cased", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 36.251, + "fp32_ovms": 35.522, + "int8_ov": 101.305, + "int8_ovms": 99.886 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i9-11900K", + "Model": "efficientdet-d0", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 59.247, + "fp32_ovms": 55.715, + "int8_ov": 148.119, + "int8_ovms": 131.749 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i9-11900K", + "Model": "manual_yolo11", + "featured_SKU": false, + "whats_new_model": true, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 85.883, + "fp32_ovms": 76.288, + "int8_ov": "", + "int8_ovms": "" + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i9-11900K", + "Model": "mask_rcnn_resnet50_atrous_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 0.37, + "fp32_ovms": 0.396, + "int8_ov": 1.321, + "int8_ovms": 1.337 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i9-11900K", + "Model": "mobilenet-v2", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 855.526, + "fp32_ovms": 731.031, + "int8_ov": 2745.282, + "int8_ovms": 2154.044 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i9-11900K", + "Model": "resnet-50", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 115.109, + "fp32_ovms": 112.697, + "int8_ov": 455.027, + "int8_ovms": 439.19 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i9-11900K", + "Model": "ssd-resnet34-1200", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 2.004, + "fp32_ovms": 2.027, + "int8_ov": 7.796, + "int8_ovms": 7.748 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i9-11900K", + "Model": "ssd_mobilenet_v1_coco", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 274.523, + "fp32_ovms": 263.584, + "int8_ov": 966.639, + "int8_ovms": 916.111 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + }, + { + "Platform": "Intel® Core™ i9-11900K", + "Model": "yolo_v8n", + "featured_SKU": false, + "whats_new_model": false, + "PlatformType": "Client Platforms (Intel® Core™)", + "Parameters": { + "throughput": { + "Precisions": [ + { + "fp32_ov": 74.006, + "fp32_ovms": 67.401, + "int8_ov": 204.296, + "int8_ovms": 151.665 + } + ], + "Unit": "FPS", + "UnitDesc": "higher is better" + } + } + } ] \ No newline at end of file diff --git a/docs/sphinx_setup/_static/benchmarks_files/graph-config.json b/docs/sphinx_setup/_static/benchmarks_files/graph-config.json index e5fe953b72bca1..e090e5abe97474 100644 --- a/docs/sphinx_setup/_static/benchmarks_files/graph-config.json +++ b/docs/sphinx_setup/_static/benchmarks_files/graph-config.json @@ -123,11 +123,11 @@ "platformTypes": { "name": "ietype", "data": [ - "None", "Intel® Core™, CPU-only", "Intel® Core™, iGPU-only", "Intel® Core™, NPU-only", - "Intel® Core™, CPU+iGPU" + "Intel® Core™, CPU+iGPU", + "Server Platforms (Intel® Xeon®)" ] }, "platforms": { diff --git a/docs/sphinx_setup/_static/download/GenAI_Quick_Start_Guide.pdf b/docs/sphinx_setup/_static/download/GenAI_Quick_Start_Guide.pdf new file mode 100644 index 00000000000000..5b6178d85c504b Binary files /dev/null and b/docs/sphinx_setup/_static/download/GenAI_Quick_Start_Guide.pdf differ diff --git a/docs/sphinx_setup/_static/js/graphs.js b/docs/sphinx_setup/_static/js/graphs.js index f29042de3e51b7..697911bad9402c 100644 --- a/docs/sphinx_setup/_static/js/graphs.js +++ b/docs/sphinx_setup/_static/js/graphs.js @@ -58,10 +58,11 @@ class Filter { return kpis; } // param: GraphData[], clientPlatforms[] - static ByClientPlatforms(graphDataArr, platformsArr) { - return graphDataArr.filter((data) => { - return platformsArr.includes(data.Platform) - }); + static BySortPlatforms(graphDataArr, platformsArr) { + return graphDataArr + .filter((data) => platformsArr.includes(data.Platform)) + .sort((a, b) => a.Platform.localeCompare(b.Platform)); + //sort is necessary } } @@ -100,13 +101,14 @@ class Graph { .sort((a, b) => a.localeCompare(b)); } static getIeTypes(graphDataArr) { - return Array.from(new Set(graphDataArr.map((obj) => obj.PlatformType))).sort((a, b) => a.localeCompare(b)); + return Array.from(new Set(graphDataArr.map((obj) => obj.PlatformType))) + .sort((a, b) => a.localeCompare(b)); } // param: GraphData[] static getPlatformNames(graphDataArr) { return graphDataArr.map((data) => data.Platform) - .sort((a, b) => a.localeCompare(b)); + .sort((a, b) => a.localeCompare(b)); } // param: GraphData[], engine: string, precisions: list @@ -659,17 +661,16 @@ $(document).ready(function () { var filteredNetworkModels = Filter.FilterByNetworkModel(graph, [networkModel]); var filteredIeTypes = Filter.ByIeTypes(filteredNetworkModels, ieTypes); - var filteredGraphData = Filter.ByClientPlatforms(filteredIeTypes, platforms); + var filteredGraphData = Filter.BySortPlatforms(filteredIeTypes, platforms); $('.chart-placeholder').append(chartContainer); - var labels = Graph.getPlatformNames(filteredGraphData); if (filteredGraphData.length > 0) { if (isLLM === true) { var graphConfigs = setGraphConfigsByEngines(filteredGraphData, appConfig, kpis, precisions) - createChartWithNewDataByEngines(labels, graphConfigs, chartContainer, display); + createChartWithNewDataByEngines(platforms, graphConfigs, chartContainer, display); } else { var graphConfigs = setGraphConfigs(filteredGraphData, appConfig, kpis, precisions) - createChartWithNewData(labels, graphConfigs, appConfig, chartContainer, display); + createChartWithNewData(platforms, graphConfigs, appConfig, chartContainer, display); } } else { diff --git a/docs/sphinx_setup/_static/selector-tool/assets/selector-Bu10eOtw.js b/docs/sphinx_setup/_static/selector-tool/assets/selector-ww24l5P1.js similarity index 84% rename from docs/sphinx_setup/_static/selector-tool/assets/selector-Bu10eOtw.js rename to docs/sphinx_setup/_static/selector-tool/assets/selector-ww24l5P1.js index 1201e390e5c7c6..24bb42f7f391d6 100644 --- a/docs/sphinx_setup/_static/selector-tool/assets/selector-Bu10eOtw.js +++ b/docs/sphinx_setup/_static/selector-tool/assets/selector-ww24l5P1.js @@ -58,4 +58,4 @@ enabled=1 gpgcheck=1 repo_gpgcheck=1 gpgkey=https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB -EOF`,getMoveRepoFileCommand:e=>`sudo mv /tmp/openvino-${e.metadata.yumYear}.repo ${oc}`,verifyRepoCommand:"yum repolist | grep -i openvino",getInstallCommand:e=>`sudo yum install openvino-${e.metadata.yumVersion}`};class jv extends De{constructor(t){super({level:j.DISTRIBUTION,key:A.ZYPPER,metadata:{title:"ZYPPER",subtitle:Z("distributions.CAPIOnly")}}),this._data=t}get data(){return{...this._data,commands:Iv}}}const Iv={addRepo:"sudo zypper addrepo https://download.opensuse.org/repositories/science/openSUSE_Tumbleweed/science.repo",refresh:"sudo zypper refresh",getInstallCommand:({metadata:e})=>`sudo zypper install openvino-devel-${e.zypperVersion} openvino-sample-${e.zypperVersion}`};class Uf extends Ie{constructor(t,n,r){super({level:j.PACKAGE,key:t,metadata:n,childrenSelector:Ff},r),this._setDefaultPackage()}_setDefaultPackage(){const t=Ne.OPENVINO_BASE;this.key===t&&this.default()}}class Rv extends Uf{constructor(t){super(Ne.OPENVINO_BASE,{title:Z("package.base.title"),subtitle:Z("package.base.subtitle")},t)}}class Tv extends Uf{constructor(t){super(Ne.OPENVINO_GENAI,{title:Z("package.genai.title"),subtitle:Z("package.genai.subtitle")},t)}}class jo extends Ie{constructor(t,n,r){super({level:j.VERSION,key:t,metadata:n},r)}}const Lv={title:Z("versions.titles.nightlyBuild"),pipVersion:"",githubVersion:"master",giteeVersion:"master",genaiGitVersion:"master",systemRequirementsLink:"https://docs.openvino.ai/nightly/about-openvino/release-notes-openvino/system-requirements.html",getStartedLink:"https://docs.openvino.ai/nightly/get-started.html",troubleshootingLink:"https://docs.openvino.ai/nightly/get-started/troubleshooting-install-config.html"};class zf extends jo{constructor(t){super(Ut.NIGHTLY,Lv,t)}}const Av={title:"2024.5",subtitle:Z("versions.titles.recommended"),pipVersion:"2024.5.0",githubVersion:"2024.5.0",giteeVersion:"2024.5.0",genaiGitVersion:"releases/2024/5",aptYear:2024,aptVersion:"2024.5.0",yumYear:2024,yumVersion:"2024.5.0",condaVersion:"2024.5.0",conanVersion:"2024.5.0",npmVersion:"2024.5.0",zypperVersion:"2024.5.0",systemRequirementsLink:"https://docs.openvino.ai/2024/about-openvino/system-requirements.html",getStartedLink:"https://docs.openvino.ai/2024/get-started.html",troubleshootingLink:"https://docs.openvino.ai/2024/get-started/troubleshooting-install-config.html"};class Vf extends jo{constructor(t){super(Ut.v_2024_5_0,Av,t)}}const Fv={title:`2023.3 ${Z("versions.titles.LTS")}`,pipVersion:"2023.3.0",githubVersion:"2023.3.0",giteeVersion:"2023.3.0",aptYear:2023,aptVersion:"2023.3.0",yumYear:2023,yumVersion:"2023.3.0",condaVersion:"2023.3.0",conanVersion:"2023.3.0",systemRequirementsLink:"https://docs.openvino.ai/2023.3/system_requirements.html",getStartedLink:"https://docs.openvino.ai/2023.3/get_started.html",troubleshootingLink:"https://docs.openvino.ai/2023.3/openvino_docs_get_started_guide_troubleshooting.html"};class Dv extends jo{constructor(t){super(Ut.v_2023_3_0,Fv,t)}}const Uv={title:`2022.3.2 ${Z("versions.titles.LTS")}`,subtitle:Z("versions.titles.hddlSupport"),pipVersion:"2022.3.2",githubVersion:"2022.3.2",giteeVersion:"2022.3.2",aptYear:2022,aptVersion:"2022.3.2",yumYear:2022,yumVersion:"2022.3.2",systemRequirementsLink:"https://docs.openvino.ai/systemrequirements",getStartedLink:"https://docs.openvino.ai/2022.3/get_started.html",troubleshootingLink:"https://docs.openvino.ai/2022.3/openvino_docs_get_started_guide_troubleshooting_steps.html"};class bf extends jo{constructor(t){super(Ut.v_2022_3_2,Uv,t)}}class ha extends Ie{constructor(t,n,r){super({level:j.OP_SYSTEM,key:t,metadata:n,childrenSelector:Ff},r),this._setDefaultOS()}_setDefaultOS(){const t=this._detectOS()||Je.WINDOWS;this.key===t&&this.default()}_detectOS(){const{userAgent:t}=navigator,n={windows:/(Windows|Win)/g,macOS:/(Macintosh|Mac)/g,linux:/(Linux|X11)/g};return n.windows.test(t)?Je.WINDOWS:n.macOS.test(t)?Je.MACOS:n.linux.test(t)?Je.LINUX:null}}class On extends ha{constructor(t){super(Je.WINDOWS,tv,t)}}class _n extends ha{constructor(t){super(Je.MACOS,nv,t)}}class Nn extends ha{constructor(t){super(Je.LINUX,rv,t)}}const zv=new Nn([new ee({linksSet:{installation:"https://docs.openvino.ai/nightly/get-started/install-openvino/install-openvino-pip.html"},frameworks:[]},{pythonAPI:!0}).includesNPUPlugin().default(),new fe({linksSet:{installation:"https://docs.openvino.ai/nightly/get-started/install-openvino/install-openvino-archive-linux.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/latest"}),new ie,new re]),Vv=new _n([new ee({linksSet:{installation:"https://docs.openvino.ai/nightly/get-started/install-openvino/install-openvino-pip.html"},frameworks:[]},{pythonAPI:!0}).default(),new fe({linksSet:{installation:"https://docs.openvino.ai/nightly/get-started/install-openvino/install-openvino-archive-macos.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/latest"}),new ie,new re]),bv=new On([new ee({linksSet:{installation:"https://docs.openvino.ai/nightly/get-started/install-openvino/install-openvino-pip.html"},frameworks:[]},{pythonAPI:!0}).includesNPUPlugin().default(),new fe({linksSet:{installation:"https://docs.openvino.ai/nightly/get-started/install-openvino/install-openvino-archive-windows.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/latest"}),new ie,new re]),Mv=new zf([bv,Vv,zv]);class $v extends Ie{constructor(t){super({level:j.ENVIRONMENT,key:ri.RUNTIME,metadata:{title:Z("environment.runtime.title"),subtitle:Z("environment.runtime.subtitle")}},t)}}class Bv extends Ie{constructor(t){super({level:j.ENVIRONMENT,key:ri.DEV_TOOLS,metadata:{title:Z("environment.devTools.title"),subtitle:Z("environment.devTools.subtitle")}},t)}}const Kv=new Nn([new ee({linksSet:{installation:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_install_dev_tools.html"},frameworks:[]},{hasFrameworks:!0}).default(),new ie,new re,new zt({linksSet:{installation:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_installing_openvino_docker_linux.html"},downloadLink:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_installing_openvino_docker_linux.html"})]),Hv=new _n([new ee({linksSet:{installation:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_install_dev_tools.html"},frameworks:[]},{hasFrameworks:!0}).addFootnote(j.OP_SYSTEM).default(),new ie().addFootnote(j.OP_SYSTEM),new re().addFootnote(j.OP_SYSTEM)]),Wv=new On([new ee({linksSet:{installation:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_install_dev_tools.html"},frameworks:[]},{hasFrameworks:!0}).default(),new ie,new re]),Gv=new Bv([Wv,Hv,Kv]),Yv=new Nn([new ee({linksSet:{installation:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_installing_openvino_pip.html"},frameworks:[]},{pythonAPI:!0}).default(),new fe({linksSet:{installation:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_installing_openvino_from_archive_linux.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.3.2/linux"}),new ca({linksSet:{installation:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_installing_openvino_apt.html"},os:[te.UBUNTU_18,te.UBUNTU_20]}),new pa({linksSet:{installation:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_installing_openvino_yum.html"}}),new ie,new re,new zt({linksSet:{installation:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_installing_openvino_docker_linux.html"},downloadLink:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_installing_openvino_docker_linux.html"})]),Qv=new _n([new ee({linksSet:{installation:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_installing_openvino_pip.html"},frameworks:[]},{pythonAPI:!0}).addFootnote(j.OP_SYSTEM).default(),new fe({linksSet:{installation:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_installing_openvino_from_archive_macos.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.3.2/macos"}).addFootnote(j.OP_SYSTEM),new ie().addFootnote(j.OP_SYSTEM),new re().addFootnote(j.OP_SYSTEM)]),Xv=new On([new ee({linksSet:{installation:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_installing_openvino_pip.html"},frameworks:[]},{pythonAPI:!0}).default(),new fe({linksSet:{installation:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_installing_openvino_from_archive_windows.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.3.2/windows"}),new ie,new re,new zt({linksSet:{installation:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_installing_openvino_docker_windows.html"},downloadLink:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_installing_openvino_docker_windows.html"})]),Jv=new $v([Xv,Qv,Yv]),Zv=new bf([Gv.default(),Jv]),qv=new Nn([new ee({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_pip.html"},frameworks:[]},{pythonAPI:!0}).default(),new fe({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_from_archive_linux.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.3/linux"}).includesNPUPlugin(),new ca({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_apt.html"},os:[te.UBUNTU_18,te.UBUNTU_20,te.UBUNTU_22]}),new pa({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_yum.html"}}),new ie,new re,new zt({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_docker.html"},downloadLink:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_docker.html"}),new ur({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_conda.html"}}),new ar({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_conan.html"}})]),ey=new _n([new ee({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_pip.html"},frameworks:[]},{pythonAPI:!0}).default(),new fe({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_from_archive_macos.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.3/macos"}),new ie,new re,new ur({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_conda.html"}}),new ar({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_conan.html"}})]),ty=new On([new ee({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_pip.html"},frameworks:[]},{pythonAPI:!0}).default(),new fe({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_from_archive_windows.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.3/windows"}).includesNPUPlugin(),new ie,new re,new zt({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_docker.html"},downloadLink:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_docker.html"}),new ur({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_conda.html"}}),new ar({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_conan.html"}})]),ny=new Dv([ty,ey,qv]),ry=new Nn([new ee({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-pip.html"},frameworks:[]},{pythonAPI:!0}).includesNPUPlugin().default(),new fe({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-archive-linux.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.5/linux"}).includesNPUPlugin(),new ca({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-apt.html"},os:[te.UBUNTU_20,te.UBUNTU_22,te.UBUNTU_24]}),new pa({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-yum.html"}}),new ie,new re,new zt({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-docker-linux.html"},downloadLink:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-docker-linux.html"}),new ur({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-conda.html"}}),new Af({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-brew.html"}}),new fa({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-vcpkg.html"}}),new ar({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-conan.html"}}),new da({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-npm.html"}}),new jv({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-zypper.html"}}),new Pv({linksSet:{installation:"https://docs.openvino.ai/2024/openvino-workflow/deployment-locally/integrate-openvino-with-ubuntu-snap.html"},downloadLink:"https://docs.openvino.ai/2024/openvino-workflow/deployment-locally/integrate-openvino-with-ubuntu-snap.html"})]),iy=new _n([new ee({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-pip.html"},frameworks:[]},{pythonAPI:!0}).default(),new fe({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-archive-macos.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.5/macos"}),new ie,new re,new ur({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-conda.html"}}),new Af({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-brew.html"}}),new fa({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-vcpkg.html"}}),new ar({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-conan.html"}}),new da({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-npm.html"}})]),oy=new On([new ee({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-pip.html"},frameworks:[]},{pythonAPI:!0}).includesNPUPlugin().default(),new fe({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-archive-windows.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.5/windows"}).includesNPUPlugin(),new ie,new re,new zt({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-docker-linux.html"},downloadLink:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-docker-linux.html"}),new ur({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-conda.html"}}),new fa({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-vcpkg.html"}}),new ar({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-conan.html"}}),new da({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-npm.html"}})]),sy=new Vf([oy,iy,ry]),ly=new Rv([sy.default(),Mv,ny,Zv]),ay=new Nn([new ee({linksSet:{installation:"https://docs.openvino.ai/nightly/get-started/install-openvino/install-openvino-genai.html#pypi-installation"},frameworks:[]},{pythonAPI:!0}).includesNPUPlugin().default(),new fe({linksSet:{installation:"https://docs.openvino.ai/nightly/get-started/install-openvino/install-openvino-genai.html#archive-installation"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/nightly/latest"}),new ie,new re]),uy=new _n([new ee({linksSet:{installation:"https://docs.openvino.ai/nightly/get-started/install-openvino/install-openvino-genai.html#pypi-installation"},frameworks:[]},{pythonAPI:!0}).default(),new fe({linksSet:{installation:"https://docs.openvino.ai/nightly/get-started/install-openvino/install-openvino-genai.html#archive-installation"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/nightly/latest"}),new ie,new re]),cy=new On([new ee({linksSet:{installation:"https://docs.openvino.ai/nightly/get-started/install-openvino/install-openvino-genai.html#pypi-installation"},frameworks:[]},{pythonAPI:!0}).includesNPUPlugin().default(),new fe({linksSet:{installation:"https://docs.openvino.ai/nightly/get-started/install-openvino/install-openvino-genai.html#archive-installation"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/nightly/latest"}),new ie,new re]),dy=new zf([cy,uy,ay]),fy=new Nn([new ee({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-genai.html"},frameworks:[]},{pythonAPI:!0}).includesNPUPlugin().default(),new fe({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-genai.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.5/linux"}).includesNPUPlugin(),new ie,new re,new zt({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-docker-linux.html"},downloadLink:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-docker-linux.html"})]),py=new _n([new ee({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-genai.html"},frameworks:[]},{pythonAPI:!0}).default(),new fe({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-genai.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.5/macos"}),new ie,new re]),hy=new On([new ee({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-genai.html"},frameworks:[]},{pythonAPI:!0}).includesNPUPlugin().default(),new fe({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-genai.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.5/windows"}).includesNPUPlugin(),new ie,new re,new zt({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-docker-linux.html"},downloadLink:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-docker-linux.html"})]),my=new Vf([hy,py,fy]),gy=new Tv([my.default(),dy]),vy=new Ie({level:j.ROOT,key:Lf.ROOT,metadata:{title:"ROOT"}},[ly.default(),gy]).default();function yy(e,t){var i,o;if(t.key===A.DOCKER||!t.footnoteLevel)return e;const n=(i=e[t.footnoteLevel])==null?void 0:i.selected,r=(o=e[t.footnoteLevel])==null?void 0:o.nodes;return!n||!r||((Array.isArray(n)?[...n,...r]:[n]).forEach(s=>s.hasFootnote=!0),r.forEach(s=>s.checked&&(s.hasFootnote=!0))),e}class wy{constructor(){Ve(this,"_root",vy)}getState(){try{return this._getState()}catch(t){return console.error(t),this._selectDefaults(this._root),this._getState()}}_getState(){const t=this._root.children,n=this._get_selected(t),r=n.children,i=this._get_selected(r),{environments:o,environment:s,systems:l,system:a}=this._processVersion(i),u=a.children,p=this._get_selected(u),d=this._processDistribution(p),h={[j.PACKAGE]:{nodes:t.map(v=>v.toOption()),selected:n.toOption()},[j.VERSION]:{nodes:r.map(v=>v.toOption()),selected:i.toOption()},[j.ENVIRONMENT]:s&&o?{nodes:o.map(v=>v.toOption()),selected:s.toOption()}:null,[j.OP_SYSTEM]:{nodes:l.map(v=>v.toOption()),selected:a.toOption()},[j.DISTRIBUTION]:{nodes:u.map(v=>v.toOption()),selected:p.toOption()},[j.FRAMEWORK]:d!=null&&d.length?{nodes:d.map(v=>v.toOption()),selected:d.filter(({checked:v})=>v).map(v=>v.toOption())}:null};return yy(h,p)}_get_selected(t){t.some(({checked:r})=>r)||this._selectDefaultsForLevel(t[0].level);const n=t.find(({checked:r})=>r);if(!n)throw new Error("Not valid tree");return n}_processVersion(t){if(t instanceof bf){const i=t.children,o=this._get_selected(i),s=o.children,l=this._get_selected(s);return{environments:i,environment:o,systems:s,system:l}}const n=t.children,r=this._get_selected(n);return{environments:null,environment:null,systems:n,system:r}}_processDistribution(t){return t instanceof ee?t.children:null}setState(t){this._setState(t)}_setState(t,n=this._root){if(!n.children.length)return;const r=n.children[0].level,i=ky(t[r]);n.children.forEach(o=>o.checked=i.includes(o.key)),n.children.forEach(o=>this._setState(t,o))}select(t,n){return this._select(t,n),this.getState()}_select(t,n,r=this._root){var i;if(((i=r.children[0])==null?void 0:i.level)!==t){r.children.forEach(o=>this._select(t,n,o));return}if(r.childrenSelector){r.childrenSelector(r.children,n);return}r.children.forEach(o=>o.checked=o.key===n)}_selectDefaultsForLevel(t,n=this._root){if(n.children.length){if(n.children[0].level!==t){n.children.forEach(r=>this._selectDefaultsForLevel(t,r));return}this._selectDefaults(n)}}_selectDefaults(t){t.children.forEach(n=>{n.checked=n.isDefault,this._selectDefaults(n)})}}const Tn=new wy;function ky(e){const t=[];return Array.isArray(e)?t.push(...e):e&&t.push(e),t}function Mf(e,{serializeVersion:t}={serializeVersion:!0}){var i,o;const n=[[j.PACKAGE,e.PACKAGE.selected.key],[j.VERSION,t?e.VERSION.selected.key:null],[j.ENVIRONMENT,(i=e.ENVIRONMENT)==null?void 0:i.selected.key],[j.OP_SYSTEM,e.OP_SYSTEM.selected.key],[j.DISTRIBUTION,e.DISTRIBUTION.selected.key],[j.FRAMEWORK,(o=e.FRAMEWORK)==null?void 0:o.selected.map(s=>s.key).join(",")]],r=new URLSearchParams;for(const[s,l]of n)l&&r.set(s,l);return r}function $f(e){function t(r,i){const o=e.get(r);if(!o)throw new Error(`Cannot extract value for: ${r}`);if(!i[o])throw new Error(`Bad node key for: ${r}`);return i[o]}function n(r,i){const o=e.get(r);return o?o.split(",").map(l=>{if(!l||!i[l])throw new Error(`Bad node key for: ${r}`);return i[l]}):null}try{return{[j.PACKAGE]:t(j.PACKAGE,Ne),[j.VERSION]:e.has(j.VERSION)?t(j.VERSION,Ut):null,[j.ENVIRONMENT]:e.has(j.ENVIRONMENT)?t(j.ENVIRONMENT,ri):null,[j.OP_SYSTEM]:t(j.OP_SYSTEM,Je),[j.DISTRIBUTION]:t(j.DISTRIBUTION,A),[j.FRAMEWORK]:n(j.FRAMEWORK,Me)}}catch(r){return console.log(`Cannot restore state from url due to error "${r}"`),null}}function Sy(){const e=window.parent;if(!e.location.search)return null;const t=new URLSearchParams(e.location.search);return $f(t)}function xy(e,t,n,{serializeVersion:r}={serializeVersion:!0}){D.useEffect(()=>{const i=window.parent,o=Mf(t,{serializeVersion:r}).toString(),s=new URL(i.location.toString());if(!s.search){s.search=o,i.history.replaceState(null,"",s);return}s.search.slice(1)!==o&&(s.search=o,i.history.pushState(null,"",s))}),parent.onpopstate=()=>{const i=window.parent,o=new URLSearchParams(i.location.search),s=$f(o);s&&(e.setState(s),n(e.getState()))}}const us=function(e){let t,n=!1;return function(...r){return n||(t=e(r),n=!0),t}};function Oy(e){var t,n;return typeof((n=(t=e.wap_tms)==null?void 0:t.custom)==null?void 0:n.trackComponentClick)!="function"?null:e.wap_tms.custom.trackComponentClick.bind(e.wap_tms.custom)}class _y{constructor(){Ve(this,"_window");Ve(this,"_consoleNotification",{notInitialized:us(()=>console.log("Adobe analytics is not initialized")),notFound:us(()=>console.log("Adobe analytics not found on a page")),devMode:us(()=>console.log("Analytics in dev mode"))});Ve(this,"_send",t=>{if(!this._window){this._consoleNotification.notInitialized();return}const n=Mf(Tn.getState()).toString(),r=Oy(this._window);if(!r){this._consoleNotification.notFound();return}try{r(t,n)}catch(i){console.error(i)}})}initialize(t){this._window=t}install(){this._send("install")}combinationView(){this._send("combination-view")}}const Ge=new _y;function Ny(){const e=Sy();e&&Tn.setState(e);const t=D.createContext((r,i)=>{Tn.select(r,i)});function n(){const[r,i]=D.useState(Tn.getState());return xy(Tn,r,i),[r,(o,s)=>i(Tn.select(o,s))]}return{SelectorContext:t,useSelector:n}}async function Py(e){e&&(navigator.clipboard?await navigator.clipboard.writeText(e):Ey(e))}function Ey(e){const t=Cy(e);document.body.append(t),t.select(),document.execCommand("copy"),t.remove()}function Cy(e){const t=document.createElement("textarea");t.style.fontSize="12pt",t.style.border="0",t.style.padding="0",t.style.margin="0",t.style.position="absolute",t.style.left="-9999px";const n=window.pageYOffset||document.documentElement.scrollTop;return t.style.top=`${n}px`,t.setAttribute("readonly",""),t.value=e,t}function jy(){return m.jsxs("svg",{version:"1.1",width:"24",height:"24",viewBox:"0 0 205 205",xmlns:"http://www.w3.org/2000/svg",className:"svg-icon",children:[m.jsx("path",{fill:"none",stroke:"currentColor",strokeWidth:"10",d:"M 50 145 a 15 15 0 0 1 -15 -15 v -90 a 15 15 0 0 1 15 -15 h 70 a 15 15 0 0 1 15 15 v 5"}),m.jsx("rect",{x:"65",y:"60",width:"100",height:"120",rx:"15",fill:"none",stroke:"currentColor",strokeWidth:"10"})]})}function Iy(){return m.jsx("svg",{version:"1.1",width:"24",height:"24",viewBox:"0 0 200 200",xmlns:"http://www.w3.org/2000/svg",className:"svg-icon",children:m.jsx("path",{strokeLinejoin:"round",strokeLinecap:"round",fill:"none",stroke:"currentColor",strokeWidth:"15",d:"M 40 100 L 90 150 L 170 40"})})}function Ry(){return m.jsxs("svg",{version:"1.1",width:"24",height:"24",viewBox:"0 0 200 200",xmlns:"http://www.w3.org/2000/svg",className:"svg-icon",children:[m.jsx("circle",{cx:"100",cy:"100",r:"90",fill:"none",strokeWidth:"15",stroke:"currentColor"}),m.jsx("path",{fill:"none",stroke:"currentColor",strokeWidth:"15",strokeLinecap:"round",d:"M 65 80 A 35 35 0 1 1 100 115"}),m.jsx("circle",{cx:"100",cy:"150",r:"3",fill:"none",stroke:"currentColor",strokeWidth:"15"})]})}const b=({comment:e,command:t,onCopy:n})=>{const[r,i]=D.useState(!1),o=async()=>{r||(await Py(t),i(!0),setTimeout(()=>i(!1),1500),n==null||n())};return m.jsxs("div",{className:"st-code-snippet","data-cy":"instructions-step",children:[e&&m.jsx(Bf,{children:e}),m.jsxs("div",{"data-cy":"command",children:[t&&m.jsx("code",{className:"st-code-snippet-content",children:t}),t&&m.jsx("button",{className:"copy-button",type:"button","aria-label":"Copy","data-cy":"copy",onClick:o,children:r?m.jsx(Iy,{}):m.jsx(jy,{})})]})]})},Bf=({children:e})=>m.jsxs("pre",{className:"st-code-snippet-comment",children:["# ",e]}),Ty=({comment:e,snippets:t})=>m.jsxs("div",{className:"st-code-snippet-multi-line","data-cy":"command",children:[e&&m.jsx(Bf,{children:e}),t.map(n=>m.jsx(b,{...n},n.command))]});function Ly(e){return e.host==="docs.openvino.ai"}const cs="production.docs.en",Kf=(cs==null?void 0:cs.includes("idz"))||!1,ds={link:"spark-hyperlink spark-hyperlink-primary spark-hyperlink-standard spark-focus-visible spark-focus-visible-self spark-focus-visible-snap spark-focus-visible-background",button:"spark-button spark-button-action spark-button-size-m spark-focus-visible spark-focus-visible-self spark-focus-visible-snap",buttonContent:"spark-button-content"},Oe=({href:e,children:t,type:n="link",testId:r="link",onClick:i})=>{const o=!Kf&&Ly(new URL(e))?"_parent":"_blank";return n==="link"?m.jsx("a",{href:e,className:ds.link,target:o,rel:"noreferrer noopener","data-cy":r,onClick:()=>i==null?void 0:i(),children:t}):m.jsx("span",{className:ds.button,children:m.jsx("span",{className:ds.buttonContent,children:m.jsx("a",{href:e,target:o,rel:"noreferrer noopener","data-cy":r,onClick:()=>i==null?void 0:i(),children:t})})})},Ay={heading:"spark-heading spark-font-200"},Ce=({title:e,accent:t=!1,dashed:n=!1,children:r,testId:i})=>m.jsxs("div",{className:`st-section ${t?"st-section-accent":""} ${n?"st-section-dashed":""}`,"data-cy":i,children:[m.jsx("span",{className:`st-section-title ${Ay.heading}`,children:e}),m.jsx("div",{className:"st-section-content",children:D.Children.map(r,o=>m.jsx(Fy,{children:o}))})]}),Fy=({children:e})=>m.jsx("div",{className:"st-section-content-row",children:e}),Dy=({version:e,distribution:t})=>{const{t:n}=$("translation",{keyPrefix:"distributions.apt.steps"}),{t:r}=$("translation",{keyPrefix:"selectorForm.titles"}),{commands:i}=t.data,o={comment:m.jsxs(z,{ns:"translation",i18nKey:"distributions.apt.steps.addRepository",children:[m.jsx("b",{children:"Step 3:"})," Add the repository via the following command"]}),snippets:i.getAddRepositoryCommands(e,t.data.os).map(({ubuntuVersionNumber:l,command:a})=>({comment:`Ubuntu ${l}`,command:a}))},s={downloadKey:{comment:m.jsxs(z,{t:n,i18nKey:"download",values:{filename:i.keyFilename},children:[m.jsx("b",{children:"Step 1:"})," Download the ",m.jsx(Oe,{href:i.keyHref,children:i.keyFilename}),". You can also use the following command"]}),command:i.downloadKeyCommand},addKey:{comment:m.jsxs(z,{t:n,i18nKey:"addKey",children:[m.jsx("b",{children:"Step 2:"})," Add this key to the system keyring"]}),command:i.addKeyCommand},addRepository:o,updatePackages:{comment:m.jsxs(z,{t:n,i18nKey:"updateList",children:[m.jsx("b",{children:"Step 4:"})," Update the list of packages via the update command"]}),command:i.updatePackagesCommand},verifyAptCache:{comment:m.jsxs(z,{t:n,i18nKey:"verify",children:[m.jsx("b",{children:"Step 5:"})," Verify that the APT repository is properly set up. Run the apt-cache command to see a list of all available OpenVINO packages and components"]}),command:i.verifyAptCacheCommand},install:{comment:m.jsxs(z,{t:n,i18nKey:"install",children:[m.jsx("b",{children:"Step 6:"})," Install OpenVINO Runtime"]}),command:i.getInstallCommand(e),onCopy:()=>Ge.install()}};return m.jsxs(Ce,{title:r("install"),accent:!0,testId:"instructions",children:[m.jsx(b,{...s.downloadKey}),m.jsx(b,{...s.addKey}),m.jsx(Ty,{...s.addRepository}),m.jsx(b,{...s.updatePackages}),m.jsx(b,{...s.verifyAptCache}),m.jsx(b,{...s.install})]})},Uy=({distribution:e})=>{const{t}=$("translation",{keyPrefix:"distributions.brew.steps"}),{t:n}=$("translation",{keyPrefix:"selectorForm.titles"}),{commands:r}=e.data,i={install:{comment:m.jsx(z,{t,i18nKey:"install",children:"Download and install the package"}),command:r.install,onCopy:()=>Ge.install()}};return m.jsx(Ce,{title:n("install"),accent:!0,testId:"instructions",children:m.jsx(b,{...i.install})})},zy=({version:e,distribution:t})=>{const{t:n}=$("translation",{keyPrefix:"distributions.conan.steps"}),{t:r}=$("translation",{keyPrefix:"selectorForm.titles"}),{commands:i}=t.data,{txtFilename:o,cmakeFilename:s}=i,l={createConanFile:{comment:m.jsxs(z,{t:n,i18nKey:"createConanFile",values:{txtFilename:o},children:[m.jsx("b",{children:"Step 1:"})," Create a ",m.jsx("b",{children:o})," file for your OpenVINO project and add “openvino” dependency in there"]}),command:i.conanTXTContent(e)},install:{comment:m.jsxs(z,{t:n,i18nKey:"install",values:{cmakeFilename:s},children:[m.jsx("b",{children:"Step 2:"})," Run the command below to create ",m.jsx("b",{children:s})," file, which will be used to compile your project with OpenVINO"]}),command:i.install,onCopy:()=>Ge.install()},compile:{comment:m.jsxs(z,{t:n,i18nKey:"compile",children:[m.jsx("b",{children:"Step 3:"})," Configure and compile your project with OpenVINO"]}),command:i.compile}};return m.jsxs(Ce,{title:r("install"),accent:!0,testId:"instructions",children:[m.jsx(b,{...l.createConanFile}),m.jsx(b,{...l.install}),m.jsx(b,{...l.compile})]})},Vy=({version:e,distribution:t})=>{const{t:n}=$("translation",{keyPrefix:"distributions.conda.steps"}),{t:r}=$("translation",{keyPrefix:"selectorForm.titles"}),{commands:i}=t.data,o={createEnv:{comment:m.jsxs(z,{t:n,i18nKey:"createEnv",children:[m.jsx("b",{children:"Step 1:"})," Create the Anaconda environment (Python 3.10 used as an example)"]}),command:i.createEnv},activateEnv:{comment:m.jsxs(z,{t:n,i18nKey:"activateEnv",children:[m.jsx("b",{children:"Step 2:"})," Activate the Anaconda environment"]}),command:i.activateEnv},upgradePip:{comment:m.jsxs(z,{t:n,i18nKey:"update",children:[m.jsx("b",{children:"Step 3:"})," Update the Anaconda to latest version"]}),command:i.update},install:{comment:m.jsxs(z,{t:n,i18nKey:"install",children:[m.jsx("b",{children:"Step 4:"})," Download and install the package"]}),command:i.getInstall(e),onCopy:()=>Ge.install()}};return m.jsxs(Ce,{title:r("install"),accent:!0,testId:"instructions",children:[m.jsx(b,{...o.createEnv}),m.jsx(b,{...o.activateEnv}),m.jsx(b,{...o.upgradePip}),m.jsx(b,{...o.install})]})},fs=({ovPackage:e,distribution:t})=>{const{t:n}=$("translation",{keyPrefix:"distributions.download"}),{t:r}=$("translation",{keyPrefix:"selectorForm.titles"}),i={[A.ARCHIVE]:e.key===Ne.OPENVINO_BASE?n("downloadArchives"):n("downloadArchivesGenAI"),[A.DOCKER]:n("gotoDocker"),[A.SNAP]:n("gotoInstallInstruction")}[t.key],o=m.jsxs(m.Fragment,{children:[n("useFollowingLink"),m.jsx("br",{}),m.jsx("b",{children:m.jsx(Oe,{href:t.data.downloadLink,testId:"download-button",onClick:()=>Ge.install(),children:i})})]});return m.jsx(Ce,{title:r("install"),accent:!0,testId:"instructions",children:m.jsx(b,{comment:o})})},by=({ovPackage:e,version:t,distribution:n})=>{const{t:r}=$("translation",{keyPrefix:"distributions.githubGitee"}),{t:i}=$("translation",{keyPrefix:"selectorForm.titles"}),o={clone:{comment:m.jsxs(z,{t:r,i18nKey:"steps.useGitClone",children:[m.jsx("b",{children:"Step 1:"})," Use Git to clone the OpenVINO toolkit repository"]}),command:n.data.commands.getCloneCommand(e,t),onCopy:()=>Ge.install()},build:{comment:m.jsxs(z,{t:r,i18nKey:"steps.buildInstructions",children:[m.jsx("b",{children:"Step 2:"})," Follow the ",m.jsx(Oe,{href:n.data.links.getBuildInstructionsLink(e,t),testId:"build-instructions-link",children:"instructions to build from source"})]})}};return m.jsxs(Ce,{title:i("install"),accent:!0,testId:"instructions",children:[m.jsx(b,{...o.clone}),m.jsx(b,{...o.build})]})},My=({distribution:e,version:t})=>{const{t:n}=$("translation",{keyPrefix:"distributions.npm.steps"}),{t:r}=$("translation",{keyPrefix:"selectorForm.titles"}),{commands:i}=e.data,o={install:{comment:m.jsx(z,{t:n,i18nKey:"install",children:"Download and install the package"}),command:i.getInstall(t),onCopy:()=>Ge.install()}};return m.jsx(Ce,{title:r("install"),accent:!0,testId:"instructions",children:m.jsx(b,{...o.install})})},$y=({ovPackage:e,environment:t,os:n,version:r,distribution:i})=>{const{t:o}=$("translation",{keyPrefix:"distributions.pip.steps"}),{t:s}=$("translation",{keyPrefix:"selectorForm.titles"}),{commands:l}=i.data,a=l.getCreateVenvCommand(n,r),u=l.getActivateVenvCommand(n,r),p=l.getInstallCommand({ovPackage:e,environment:t,os:n,version:r,distribution:i}),d={createEnv:{comment:m.jsxs(z,{t:o,i18nKey:"createVenv",children:[m.jsx("b",{children:"Step 1:"})," Create virtual environment"]}),command:a},activateEnv:{comment:m.jsxs(z,{t:o,i18nKey:"activateVenv",children:[m.jsx("b",{children:"Step 2:"})," Activate virtual environment"]}),command:u},upgradePip:{comment:m.jsxs(z,{t:o,i18nKey:"upgradePip",children:[m.jsx("b",{children:"Step 3:"})," Upgrade pip to latest version"]}),command:l.upgradeCommand},install:{comment:m.jsxs(z,{t:o,i18nKey:"install",children:[m.jsx("b",{children:"Step 4:"})," Download and install the package"]}),command:p,onCopy:()=>Ge.install()}};return m.jsxs(Ce,{title:s("install"),accent:!0,testId:"instructions",children:[m.jsx(b,{...d.createEnv}),m.jsx(b,{...d.activateEnv}),m.jsx(b,{...d.upgradePip}),m.jsx(b,{...d.install})]})},By=({distribution:e})=>{const{t}=$("translation",{keyPrefix:"distributions.vcpkg.steps"}),{t:n}=$("translation",{keyPrefix:"selectorForm.titles"}),{commands:r}=e.data,i={install:{comment:m.jsx(z,{t,i18nKey:"install",children:"Download and install the package"}),command:r.install,onCopy:()=>Ge.install()}};return m.jsx(Ce,{title:n("install"),accent:!0,testId:"instructions",children:m.jsx(b,{...i.install})})},Ky=({version:e,distribution:t})=>{const{t:n}=$("translation",{keyPrefix:"distributions.yum.steps"}),{t:r}=$("translation",{keyPrefix:"selectorForm.titles"}),{yumYear:i}=e.metadata,{commands:o}=t.data,s={createRepo:{comment:m.jsxs(z,{t:n,i18nKey:"createRepoFile",children:[m.jsx("b",{children:"Step 1:"})," Create the YUM repo file in the /tmp directory as a normal user"]}),command:o.getCreateRepoCommand(e)},moveRepoFile:{comment:m.jsxs(z,{t:n,i18nKey:"moveFile",values:{year:i,directory:o.directory},children:[m.jsx("b",{children:"Step 2:"})," Move the new openvino-",{year:i},".repo file to the YUM configuration directory ",m.jsx("b",{children:o.directory})]}),command:o.getMoveRepoFileCommand(e)},verifyRepo:{comment:m.jsxs(z,{t:n,i18nKey:"verify",children:[m.jsx("b",{children:"Step 3:"})," Verify that the new repo is properly setup by running the following command"]}),command:o.verifyRepoCommand},install:{comment:m.jsxs(z,{t:n,i18nKey:"install",children:[m.jsx("b",{children:"Step 4:"})," Install OpenVINO Runtime"]}),command:o.getInstallCommand(e),onCopy:()=>Ge.install()}};return m.jsxs(Ce,{title:r("install"),accent:!0,testId:"instructions",children:[m.jsx(b,{...s.createRepo}),m.jsx(b,{...s.moveRepoFile}),m.jsx(b,{...s.verifyRepo}),m.jsx(b,{...s.install})]})},Hy=({version:e,distribution:t})=>{const{t:n}=$("translation",{keyPrefix:"distributions.zypper.steps"}),{t:r}=$("translation",{keyPrefix:"selectorForm.titles"}),{commands:i}=t.data,o={addRepo:{comment:m.jsxs(z,{t:n,i18nKey:"addRepo",children:[m.jsx("b",{children:"Step 1:"})," Create a ZYPPER repository file with the command below"]}),command:i.addRepo},refresh:{comment:m.jsxs(z,{t:n,i18nKey:"refresh",children:[m.jsx("b",{children:"Step 2:"})," Refresh repositories"]}),command:i.refresh},install:{comment:m.jsxs(z,{t:n,i18nKey:"install",children:[m.jsx("b",{children:"Step 3:"})," Install OpenVINO"]}),command:i.getInstallCommand(e),onCopy:()=>Ge.install()}};return m.jsxs(Ce,{title:r("install"),accent:!0,testId:"instructions",children:[m.jsx(b,{...o.addRepo}),m.jsx(b,{...o.refresh}),m.jsx(b,{...o.install})]})},Wy=({state:e})=>{var r;const t={ovPackage:e.PACKAGE.selected,environment:(r=e.ENVIRONMENT)==null?void 0:r.selected,os:e.OP_SYSTEM.selected,version:e.VERSION.selected,distribution:e.DISTRIBUTION.selected};if(t.distribution.key===A.PIP)return m.jsx($y,{...t,distribution:t.distribution});if(t.distribution.key===A.ARCHIVE)return m.jsx(fs,{...t,distribution:t.distribution});if(t.distribution.key===A.DOCKER)return m.jsx(fs,{...t,distribution:t.distribution});if(t.distribution.key===A.GITHUB||t.distribution.key===A.GITEE)return m.jsx(by,{...t,distribution:t.distribution});if(t.distribution.key===A.APT)return m.jsx(Dy,{...t,distribution:t.distribution});if(t.distribution.key===A.YUM)return m.jsx(Ky,{...t,distribution:t.distribution});if(t.distribution.key===A.CONDA)return m.jsx(Vy,{...t,distribution:t.distribution});if(t.distribution.key===A.BREW)return m.jsx(Uy,{...t,distribution:t.distribution});if(t.distribution.key===A.VCPKG)return m.jsx(By,{...t,distribution:t.distribution});if(t.distribution.key===A.CONAN)return m.jsx(zy,{...t,distribution:t.distribution});if(t.distribution.key===A.NPM)return m.jsx(My,{...t,distribution:t.distribution});if(t.distribution.key===A.ZYPPER)return m.jsx(Hy,{...t,distribution:t.distribution});if(t.distribution.key===A.SNAP)return m.jsx(fs,{...t,distribution:t.distribution});const n=t.distribution;throw new Error(`${n}`)};function Gy(){const{t:e}=$("common",{keyPrefix:"relatedTools"}),{t}=$("translation");return m.jsx(Ce,{title:t("selectorForm.titles.relatedTools"),testId:"relatedTools",accent:!0,dashed:!0,children:m.jsxs("div",{className:"st-related-tools-links",children:[m.jsx(Oe,{href:"https://github.com/openvinotoolkit/openvino_notebooks",testId:"notebooks-link",children:e("OpenVINONotebooks")}),m.jsx(Oe,{href:"https://huggingface.co/docs/optimum/main/intel/openvino/inference",testId:"hf_optimum-link",children:"Hugging Face + Optimum Intel"}),m.jsx("div",{children:m.jsxs(z,{t:e,i18nKey:"tokenizers",children:[m.jsx(Oe,{href:"https://docs.openvino.ai/2024/learn-openvino/llm_inference_guide/ov-tokenizers.html",testId:"openvino_tokenizers-link",children:"OpenVINO Tokenizers"}),"to streamline tokenizer conversion"]})}),m.jsx("div",{children:m.jsxs(z,{t:e,i18nKey:"nncf",children:[m.jsx(Oe,{href:"https://docs.openvino.ai/2024/openvino-workflow/model-optimization-guide/compressing-models-during-training.html",testId:"nncf-link",children:"NNCF"}),"for implementing compression algorithms on models"]})}),m.jsx("div",{children:m.jsxs(z,{t:e,i18nKey:"ovms",children:[m.jsx(Oe,{href:"https://docs.openvino.ai/2024/ovms_what_is_openvino_model_server.html",testId:"ovms-link",children:"OVMS"}),"for serving models optimized for deployment"]})})]})})}function Yy({state:e}){const t=e.PACKAGE.selected,n=e.DISTRIBUTION.selected,r=e.VERSION.selected,{t:i}=$("translation",{keyPrefix:"selectorForm.titles"}),{t:o}=$("common",{keyPrefix:"resources"});let s=m.jsx(m.Fragment,{});if(A.GITHUB===n.key||A.GITEE===n.key){const l=n.key===A.GITHUB?t.key===Ne.OPENVINO_BASE?o("githubRepository"):o("githubGenAIRepository"):t.key===Ne.OPENVINO_BASE?o("giteeRepository"):o("giteeGenAIRepository");s=m.jsxs(m.Fragment,{children:[m.jsx(Oe,{href:n.data.links.getBuildInstructionsLink(t,r),testId:"install-instructions-link",children:o("installationInstructions")}),m.jsx(Oe,{href:n.data.links.getRepositoryLink(t,r),testId:"repository-link",children:l})]})}else s=m.jsx(Oe,{href:n.data.linksSet.installation,testId:"install-instructions-link",children:o("installationInstructions")});return m.jsx(Ce,{title:i("resources"),testId:"resources",accent:!0,children:m.jsxs("div",{className:"st-resources-links",children:[m.jsxs("div",{children:[s,m.jsx(Oe,{href:"https://github.com/openvinotoolkit/openvino/releases",testId:"previous-releases-link",children:o("prevReleases")}),m.jsx(Oe,{href:r.metadata.systemRequirementsLink,testId:"system-requirements-link",children:o("systemRequirements")})]}),m.jsxs("div",{children:[m.jsx(Oe,{href:r.metadata.getStartedLink,testId:"get-started-link",children:o("getStarted")}),m.jsx(Oe,{href:r.metadata.troubleshootingLink,testId:"troubleshooting-link",children:o("troubleshooting")})]})]})})}const Oi={tooltipToggle:"spark-tooltip-toggle",tooltipPopover:"spark-tooltip spark-tooltip-size-m spark-shadow spark-tooltip-bottom",tooltipLabel:"spark-tooltip-label",tooltipArrow:"spark-tooltip-tip"},Qy=({content:e})=>{const[t,n]=D.useState(!1);return m.jsx("div",{className:"help-tooltip",children:m.jsxs("div",{className:Oi.tooltipToggle,onMouseEnter:()=>n(!0),onMouseLeave:()=>n(!1),children:[m.jsx("div",{"aria-hidden":"true",role:"img",className:"help-tooltip-icon",children:m.jsx(Ry,{})}),t&&m.jsxs("div",{className:`help-tooltip-popover ${Oi.tooltipPopover}`,role:"tooltip",children:[m.jsx("span",{className:`help-tooltip-popover-content ${Oi.tooltipLabel}`,children:e}),m.jsx("span",{className:Oi.tooltipArrow})]})]})})},ps={toggleSwitchField:"spark-fieldlabel spark-fieldlabel-size-m spark-toggle-switch spark-toggle-switch-action spark-toggle-switch-size-l",toggleSwitchInput:"spark-focus-visible spark-focus-visible-suppress spark-focus-visible-adjacent",toggleSwitchSelector:"spark-toggle-switch-selector spark-focus-visible spark-focus-visible-snap"},Xy=({label:e,checked:t=!1,onClick:n,testId:r})=>m.jsxs("label",{className:ps.toggleSwitchField,"data-cy":r,children:[m.jsx("input",{type:"checkbox",className:ps.toggleSwitchInput,role:"switch",checked:t,onChange:()=>n==null?void 0:n()}),m.jsx("span",{className:ps.toggleSwitchSelector}),m.jsx("div",{children:e})]}),cn={toggleButton:"spark-button spark-button-size-l spark-focus-visible spark-focus-visible-self spark-focus-visible-snap",toggleButtonGroup:"spark-button-group spark-button-group-orientation-horizontal spark-button-group-align-start spark-button-group-spacing-l",actionButton:"spark-button-action",secondaryButton:"spark-button-secondary",disabledButton:"spark-button-disabled",buttonContent:"spark-button-content",fontXs:"spark-font-25"},Hf=({onClick:e,checked:t=!1,disabled:n=!1,title:r,subtitle:i,value:o})=>m.jsx("button",{className:`${cn.toggleButton} ${t?cn.actionButton:cn.secondaryButton} ${n&&cn.disabledButton}`,type:"button",role:"radio","aria-checked":t,onClick:()=>e==null?void 0:e(),"data-cy":o,"aria-label":r,children:m.jsxs("span",{className:cn.buttonContent,children:[m.jsx("span",{className:"title",children:r}),i&&m.jsx("span",{className:`${cn.fontXs} subtitle`,children:i})]})}),Wf=({children:e,className:t})=>m.jsx("div",{className:`option-button-group ${t||""} ${cn.toggleButtonGroup}`,children:e});function Jy({title:e,options:t,level:n}){const r=D.useContext(ma),i=t.some(({checked:u})=>u),[o,s]=D.useState(i),l=t.map(({level:u,key:p,checked:d,metadata:h})=>m.jsx(Hf,{value:`${u}_${p}`,checked:d,title:h.title,onClick:()=>r(u,p)},p)),{t:a}=$("translation",{keyPrefix:"frameworks"});return m.jsxs(Ce,{title:e,testId:n,children:[m.jsxs(m.Fragment,{children:[m.jsx(Xy,{label:a("install"),checked:o,onClick:()=>s(!o),testId:"frameworks-toggle-switch"}),m.jsx(Qy,{content:a("tooltip")})]}),m.jsx(m.Fragment,{children:o&&m.jsx(Wf,{children:l})})]})}function wr({title:e,options:t,level:n}){const r=D.useContext(ma),i=t.map(({level:o,key:s,checked:l,metadata:a})=>m.jsx(Hf,{value:`${o}_${s}`,checked:l,title:a.title,subtitle:a.subtitle,onClick:()=>r(o,s)},s));return m.jsx(Ce,{title:e,testId:n,children:m.jsx(Wf,{children:i})})}function Zy({state:e}){var a,u;const t=e.PACKAGE.nodes,n=e.VERSION.nodes,r=(a=e.ENVIRONMENT)==null?void 0:a.nodes.map(p=>({...p,metadata:{...p.metadata,subtitle:void 0}})),i=e.OP_SYSTEM.nodes,o=e.DISTRIBUTION.nodes,s=(u=e.FRAMEWORK)==null?void 0:u.nodes;D.useEffect(()=>Ge.combinationView(),[e]);const{t:l}=$("translation",{keyPrefix:"selectorForm.titles"});return m.jsxs(m.Fragment,{children:[m.jsx(wr,{title:l("package"),options:t,level:j.PACKAGE}),m.jsx(wr,{title:l("version"),options:n,level:j.VERSION}),r&&m.jsx(wr,{title:l("envinronment"),options:r,level:j.ENVIRONMENT}),m.jsx(wr,{title:l("os"),options:i,level:j.OP_SYSTEM}),m.jsx(wr,{title:l("distribution"),options:o,level:j.DISTRIBUTION}),s&&m.jsx(Jy,{title:l("frameworks"),options:s,level:j.FRAMEWORK})]})}const{SelectorContext:ma,useSelector:qy}=Ny();Ge.initialize(window.parent);function e0(){const[e,t]=qy();return m.jsx("div",{className:`st-responsive-container ${Kf?"idz-page":""}`,children:m.jsxs(ma.Provider,{value:t,children:[m.jsx(Zy,{state:e}),m.jsx(Wy,{state:e}),m.jsx(Yy,{state:e}),m.jsx(Gy,{})]})})}vs.createRoot(document.getElementById("root")).render(m.jsx(cp.StrictMode,{children:m.jsx(e0,{})})); +EOF`,getMoveRepoFileCommand:e=>`sudo mv /tmp/openvino-${e.metadata.yumYear}.repo ${oc}`,verifyRepoCommand:"yum repolist | grep -i openvino",getInstallCommand:e=>`sudo yum install openvino-${e.metadata.yumVersion}`};class jv extends De{constructor(t){super({level:j.DISTRIBUTION,key:A.ZYPPER,metadata:{title:"ZYPPER",subtitle:Z("distributions.CAPIOnly")}}),this._data=t}get data(){return{...this._data,commands:Iv}}}const Iv={addRepo:"sudo zypper addrepo https://download.opensuse.org/repositories/science/openSUSE_Tumbleweed/science.repo",refresh:"sudo zypper refresh",getInstallCommand:({metadata:e})=>`sudo zypper install openvino-devel-${e.zypperVersion} openvino-sample-${e.zypperVersion}`};class Uf extends Ie{constructor(t,n,r){super({level:j.PACKAGE,key:t,metadata:n,childrenSelector:Ff},r),this._setDefaultPackage()}_setDefaultPackage(){const t=Ne.OPENVINO_BASE;this.key===t&&this.default()}}class Rv extends Uf{constructor(t){super(Ne.OPENVINO_BASE,{title:Z("package.base.title"),subtitle:Z("package.base.subtitle")},t)}}class Tv extends Uf{constructor(t){super(Ne.OPENVINO_GENAI,{title:Z("package.genai.title"),subtitle:Z("package.genai.subtitle")},t)}}class jo extends Ie{constructor(t,n,r){super({level:j.VERSION,key:t,metadata:n},r)}}const Lv={title:Z("versions.titles.nightlyBuild"),pipVersion:"",githubVersion:"master",giteeVersion:"master",genaiGitVersion:"master",systemRequirementsLink:"https://docs.openvino.ai/nightly/about-openvino/release-notes-openvino/system-requirements.html",getStartedLink:"https://docs.openvino.ai/nightly/get-started.html",troubleshootingLink:"https://docs.openvino.ai/nightly/get-started/troubleshooting-install-config.html"};class zf extends jo{constructor(t){super(Ut.NIGHTLY,Lv,t)}}const Av={title:"2024.5",subtitle:Z("versions.titles.recommended"),pipVersion:"2024.5.0",githubVersion:"2024.5.0",giteeVersion:"2024.5.0",genaiGitVersion:"releases/2024/5",aptYear:2024,aptVersion:"2024.5.0",yumYear:2024,yumVersion:"2024.5.0",condaVersion:"2024.5.0",conanVersion:"2024.5.0",npmVersion:"2024.5.0-0",zypperVersion:"2024.5.0",systemRequirementsLink:"https://docs.openvino.ai/2024/about-openvino/system-requirements.html",getStartedLink:"https://docs.openvino.ai/2024/get-started.html",troubleshootingLink:"https://docs.openvino.ai/2024/get-started/troubleshooting-install-config.html"};class Vf extends jo{constructor(t){super(Ut.v_2024_5_0,Av,t)}}const Fv={title:`2023.3 ${Z("versions.titles.LTS")}`,pipVersion:"2023.3.0",githubVersion:"2023.3.0",giteeVersion:"2023.3.0",aptYear:2023,aptVersion:"2023.3.0",yumYear:2023,yumVersion:"2023.3.0",condaVersion:"2023.3.0",conanVersion:"2023.3.0",systemRequirementsLink:"https://docs.openvino.ai/2023.3/system_requirements.html",getStartedLink:"https://docs.openvino.ai/2023.3/get_started.html",troubleshootingLink:"https://docs.openvino.ai/2023.3/openvino_docs_get_started_guide_troubleshooting.html"};class Dv extends jo{constructor(t){super(Ut.v_2023_3_0,Fv,t)}}const Uv={title:`2022.3.2 ${Z("versions.titles.LTS")}`,subtitle:Z("versions.titles.hddlSupport"),pipVersion:"2022.3.2",githubVersion:"2022.3.2",giteeVersion:"2022.3.2",aptYear:2022,aptVersion:"2022.3.2",yumYear:2022,yumVersion:"2022.3.2",systemRequirementsLink:"https://docs.openvino.ai/systemrequirements",getStartedLink:"https://docs.openvino.ai/2022.3/get_started.html",troubleshootingLink:"https://docs.openvino.ai/2022.3/openvino_docs_get_started_guide_troubleshooting_steps.html"};class bf extends jo{constructor(t){super(Ut.v_2022_3_2,Uv,t)}}class ha extends Ie{constructor(t,n,r){super({level:j.OP_SYSTEM,key:t,metadata:n,childrenSelector:Ff},r),this._setDefaultOS()}_setDefaultOS(){const t=this._detectOS()||Je.WINDOWS;this.key===t&&this.default()}_detectOS(){const{userAgent:t}=navigator,n={windows:/(Windows|Win)/g,macOS:/(Macintosh|Mac)/g,linux:/(Linux|X11)/g};return n.windows.test(t)?Je.WINDOWS:n.macOS.test(t)?Je.MACOS:n.linux.test(t)?Je.LINUX:null}}class On extends ha{constructor(t){super(Je.WINDOWS,tv,t)}}class _n extends ha{constructor(t){super(Je.MACOS,nv,t)}}class Nn extends ha{constructor(t){super(Je.LINUX,rv,t)}}const zv=new Nn([new ee({linksSet:{installation:"https://docs.openvino.ai/nightly/get-started/install-openvino/install-openvino-pip.html"},frameworks:[]},{pythonAPI:!0}).includesNPUPlugin().default(),new fe({linksSet:{installation:"https://docs.openvino.ai/nightly/get-started/install-openvino/install-openvino-archive-linux.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/latest"}),new ie,new re]),Vv=new _n([new ee({linksSet:{installation:"https://docs.openvino.ai/nightly/get-started/install-openvino/install-openvino-pip.html"},frameworks:[]},{pythonAPI:!0}).default(),new fe({linksSet:{installation:"https://docs.openvino.ai/nightly/get-started/install-openvino/install-openvino-archive-macos.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/latest"}),new ie,new re]),bv=new On([new ee({linksSet:{installation:"https://docs.openvino.ai/nightly/get-started/install-openvino/install-openvino-pip.html"},frameworks:[]},{pythonAPI:!0}).includesNPUPlugin().default(),new fe({linksSet:{installation:"https://docs.openvino.ai/nightly/get-started/install-openvino/install-openvino-archive-windows.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/latest"}),new ie,new re]),Mv=new zf([bv,Vv,zv]);class $v extends Ie{constructor(t){super({level:j.ENVIRONMENT,key:ri.RUNTIME,metadata:{title:Z("environment.runtime.title"),subtitle:Z("environment.runtime.subtitle")}},t)}}class Bv extends Ie{constructor(t){super({level:j.ENVIRONMENT,key:ri.DEV_TOOLS,metadata:{title:Z("environment.devTools.title"),subtitle:Z("environment.devTools.subtitle")}},t)}}const Kv=new Nn([new ee({linksSet:{installation:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_install_dev_tools.html"},frameworks:[]},{hasFrameworks:!0}).default(),new ie,new re,new zt({linksSet:{installation:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_installing_openvino_docker_linux.html"},downloadLink:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_installing_openvino_docker_linux.html"})]),Hv=new _n([new ee({linksSet:{installation:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_install_dev_tools.html"},frameworks:[]},{hasFrameworks:!0}).addFootnote(j.OP_SYSTEM).default(),new ie().addFootnote(j.OP_SYSTEM),new re().addFootnote(j.OP_SYSTEM)]),Wv=new On([new ee({linksSet:{installation:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_install_dev_tools.html"},frameworks:[]},{hasFrameworks:!0}).default(),new ie,new re]),Gv=new Bv([Wv,Hv,Kv]),Yv=new Nn([new ee({linksSet:{installation:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_installing_openvino_pip.html"},frameworks:[]},{pythonAPI:!0}).default(),new fe({linksSet:{installation:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_installing_openvino_from_archive_linux.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.3.2/linux"}),new ca({linksSet:{installation:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_installing_openvino_apt.html"},os:[te.UBUNTU_18,te.UBUNTU_20]}),new pa({linksSet:{installation:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_installing_openvino_yum.html"}}),new ie,new re,new zt({linksSet:{installation:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_installing_openvino_docker_linux.html"},downloadLink:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_installing_openvino_docker_linux.html"})]),Qv=new _n([new ee({linksSet:{installation:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_installing_openvino_pip.html"},frameworks:[]},{pythonAPI:!0}).addFootnote(j.OP_SYSTEM).default(),new fe({linksSet:{installation:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_installing_openvino_from_archive_macos.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.3.2/macos"}).addFootnote(j.OP_SYSTEM),new ie().addFootnote(j.OP_SYSTEM),new re().addFootnote(j.OP_SYSTEM)]),Xv=new On([new ee({linksSet:{installation:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_installing_openvino_pip.html"},frameworks:[]},{pythonAPI:!0}).default(),new fe({linksSet:{installation:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_installing_openvino_from_archive_windows.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.3.2/windows"}),new ie,new re,new zt({linksSet:{installation:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_installing_openvino_docker_windows.html"},downloadLink:"https://docs.openvino.ai/2022.3/openvino_docs_install_guides_installing_openvino_docker_windows.html"})]),Jv=new $v([Xv,Qv,Yv]),Zv=new bf([Gv.default(),Jv]),qv=new Nn([new ee({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_pip.html"},frameworks:[]},{pythonAPI:!0}).default(),new fe({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_from_archive_linux.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.3/linux"}).includesNPUPlugin(),new ca({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_apt.html"},os:[te.UBUNTU_18,te.UBUNTU_20,te.UBUNTU_22]}),new pa({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_yum.html"}}),new ie,new re,new zt({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_docker.html"},downloadLink:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_docker.html"}),new ur({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_conda.html"}}),new ar({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_conan.html"}})]),ey=new _n([new ee({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_pip.html"},frameworks:[]},{pythonAPI:!0}).default(),new fe({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_from_archive_macos.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.3/macos"}),new ie,new re,new ur({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_conda.html"}}),new ar({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_conan.html"}})]),ty=new On([new ee({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_pip.html"},frameworks:[]},{pythonAPI:!0}).default(),new fe({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_from_archive_windows.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.3/windows"}).includesNPUPlugin(),new ie,new re,new zt({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_docker.html"},downloadLink:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_docker.html"}),new ur({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_conda.html"}}),new ar({linksSet:{installation:"https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_conan.html"}})]),ny=new Dv([ty,ey,qv]),ry=new Nn([new ee({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-pip.html"},frameworks:[]},{pythonAPI:!0}).includesNPUPlugin().default(),new fe({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-archive-linux.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.5/linux"}).includesNPUPlugin(),new ca({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-apt.html"},os:[te.UBUNTU_20,te.UBUNTU_22,te.UBUNTU_24]}),new pa({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-yum.html"}}),new ie,new re,new zt({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-docker-linux.html"},downloadLink:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-docker-linux.html"}),new ur({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-conda.html"}}),new Af({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-brew.html"}}),new fa({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-vcpkg.html"}}),new ar({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-conan.html"}}),new da({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-npm.html"}}),new jv({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-zypper.html"}}),new Pv({linksSet:{installation:"https://docs.openvino.ai/2024/openvino-workflow/deployment-locally/integrate-openvino-with-ubuntu-snap.html"},downloadLink:"https://docs.openvino.ai/2024/openvino-workflow/deployment-locally/integrate-openvino-with-ubuntu-snap.html"})]),iy=new _n([new ee({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-pip.html"},frameworks:[]},{pythonAPI:!0}).default(),new fe({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-archive-macos.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.5/macos"}),new ie,new re,new ur({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-conda.html"}}),new Af({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-brew.html"}}),new fa({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-vcpkg.html"}}),new ar({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-conan.html"}}),new da({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-npm.html"}})]),oy=new On([new ee({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-pip.html"},frameworks:[]},{pythonAPI:!0}).includesNPUPlugin().default(),new fe({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-archive-windows.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.5/windows"}).includesNPUPlugin(),new ie,new re,new zt({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-docker-linux.html"},downloadLink:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-docker-linux.html"}),new ur({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-conda.html"}}),new fa({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-vcpkg.html"}}),new ar({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-conan.html"}}),new da({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-npm.html"}})]),sy=new Vf([oy,iy,ry]),ly=new Rv([sy.default(),Mv,ny,Zv]),ay=new Nn([new ee({linksSet:{installation:"https://docs.openvino.ai/nightly/get-started/install-openvino/install-openvino-genai.html#pypi-installation"},frameworks:[]},{pythonAPI:!0}).includesNPUPlugin().default(),new fe({linksSet:{installation:"https://docs.openvino.ai/nightly/get-started/install-openvino/install-openvino-genai.html#archive-installation"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/nightly/latest"}),new ie,new re]),uy=new _n([new ee({linksSet:{installation:"https://docs.openvino.ai/nightly/get-started/install-openvino/install-openvino-genai.html#pypi-installation"},frameworks:[]},{pythonAPI:!0}).default(),new fe({linksSet:{installation:"https://docs.openvino.ai/nightly/get-started/install-openvino/install-openvino-genai.html#archive-installation"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/nightly/latest"}),new ie,new re]),cy=new On([new ee({linksSet:{installation:"https://docs.openvino.ai/nightly/get-started/install-openvino/install-openvino-genai.html#pypi-installation"},frameworks:[]},{pythonAPI:!0}).includesNPUPlugin().default(),new fe({linksSet:{installation:"https://docs.openvino.ai/nightly/get-started/install-openvino/install-openvino-genai.html#archive-installation"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/nightly/latest"}),new ie,new re]),dy=new zf([cy,uy,ay]),fy=new Nn([new ee({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-genai.html"},frameworks:[]},{pythonAPI:!0}).includesNPUPlugin().default(),new fe({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-genai.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.5/linux"}).includesNPUPlugin(),new ie,new re,new zt({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-docker-linux.html"},downloadLink:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-docker-linux.html"})]),py=new _n([new ee({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-genai.html"},frameworks:[]},{pythonAPI:!0}).default(),new fe({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-genai.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.5/macos"}),new ie,new re]),hy=new On([new ee({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-genai.html"},frameworks:[]},{pythonAPI:!0}).includesNPUPlugin().default(),new fe({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-genai.html"},downloadLink:"https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.5/windows"}).includesNPUPlugin(),new ie,new re,new zt({linksSet:{installation:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-docker-linux.html"},downloadLink:"https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-docker-linux.html"})]),my=new Vf([hy,py,fy]),gy=new Tv([my.default(),dy]),vy=new Ie({level:j.ROOT,key:Lf.ROOT,metadata:{title:"ROOT"}},[ly.default(),gy]).default();function yy(e,t){var i,o;if(t.key===A.DOCKER||!t.footnoteLevel)return e;const n=(i=e[t.footnoteLevel])==null?void 0:i.selected,r=(o=e[t.footnoteLevel])==null?void 0:o.nodes;return!n||!r||((Array.isArray(n)?[...n,...r]:[n]).forEach(s=>s.hasFootnote=!0),r.forEach(s=>s.checked&&(s.hasFootnote=!0))),e}class wy{constructor(){Ve(this,"_root",vy)}getState(){try{return this._getState()}catch(t){return console.error(t),this._selectDefaults(this._root),this._getState()}}_getState(){const t=this._root.children,n=this._get_selected(t),r=n.children,i=this._get_selected(r),{environments:o,environment:s,systems:l,system:a}=this._processVersion(i),u=a.children,p=this._get_selected(u),d=this._processDistribution(p),h={[j.PACKAGE]:{nodes:t.map(v=>v.toOption()),selected:n.toOption()},[j.VERSION]:{nodes:r.map(v=>v.toOption()),selected:i.toOption()},[j.ENVIRONMENT]:s&&o?{nodes:o.map(v=>v.toOption()),selected:s.toOption()}:null,[j.OP_SYSTEM]:{nodes:l.map(v=>v.toOption()),selected:a.toOption()},[j.DISTRIBUTION]:{nodes:u.map(v=>v.toOption()),selected:p.toOption()},[j.FRAMEWORK]:d!=null&&d.length?{nodes:d.map(v=>v.toOption()),selected:d.filter(({checked:v})=>v).map(v=>v.toOption())}:null};return yy(h,p)}_get_selected(t){t.some(({checked:r})=>r)||this._selectDefaultsForLevel(t[0].level);const n=t.find(({checked:r})=>r);if(!n)throw new Error("Not valid tree");return n}_processVersion(t){if(t instanceof bf){const i=t.children,o=this._get_selected(i),s=o.children,l=this._get_selected(s);return{environments:i,environment:o,systems:s,system:l}}const n=t.children,r=this._get_selected(n);return{environments:null,environment:null,systems:n,system:r}}_processDistribution(t){return t instanceof ee?t.children:null}setState(t){this._setState(t)}_setState(t,n=this._root){if(!n.children.length)return;const r=n.children[0].level,i=ky(t[r]);n.children.forEach(o=>o.checked=i.includes(o.key)),n.children.forEach(o=>this._setState(t,o))}select(t,n){return this._select(t,n),this.getState()}_select(t,n,r=this._root){var i;if(((i=r.children[0])==null?void 0:i.level)!==t){r.children.forEach(o=>this._select(t,n,o));return}if(r.childrenSelector){r.childrenSelector(r.children,n);return}r.children.forEach(o=>o.checked=o.key===n)}_selectDefaultsForLevel(t,n=this._root){if(n.children.length){if(n.children[0].level!==t){n.children.forEach(r=>this._selectDefaultsForLevel(t,r));return}this._selectDefaults(n)}}_selectDefaults(t){t.children.forEach(n=>{n.checked=n.isDefault,this._selectDefaults(n)})}}const Tn=new wy;function ky(e){const t=[];return Array.isArray(e)?t.push(...e):e&&t.push(e),t}function Mf(e,{serializeVersion:t}={serializeVersion:!0}){var i,o;const n=[[j.PACKAGE,e.PACKAGE.selected.key],[j.VERSION,t?e.VERSION.selected.key:null],[j.ENVIRONMENT,(i=e.ENVIRONMENT)==null?void 0:i.selected.key],[j.OP_SYSTEM,e.OP_SYSTEM.selected.key],[j.DISTRIBUTION,e.DISTRIBUTION.selected.key],[j.FRAMEWORK,(o=e.FRAMEWORK)==null?void 0:o.selected.map(s=>s.key).join(",")]],r=new URLSearchParams;for(const[s,l]of n)l&&r.set(s,l);return r}function $f(e){function t(r,i){const o=e.get(r);if(!o)throw new Error(`Cannot extract value for: ${r}`);if(!i[o])throw new Error(`Bad node key for: ${r}`);return i[o]}function n(r,i){const o=e.get(r);return o?o.split(",").map(l=>{if(!l||!i[l])throw new Error(`Bad node key for: ${r}`);return i[l]}):null}try{return{[j.PACKAGE]:t(j.PACKAGE,Ne),[j.VERSION]:e.has(j.VERSION)?t(j.VERSION,Ut):null,[j.ENVIRONMENT]:e.has(j.ENVIRONMENT)?t(j.ENVIRONMENT,ri):null,[j.OP_SYSTEM]:t(j.OP_SYSTEM,Je),[j.DISTRIBUTION]:t(j.DISTRIBUTION,A),[j.FRAMEWORK]:n(j.FRAMEWORK,Me)}}catch(r){return console.log(`Cannot restore state from url due to error "${r}"`),null}}function Sy(){const e=window.parent;if(!e.location.search)return null;const t=new URLSearchParams(e.location.search);return $f(t)}function xy(e,t,n,{serializeVersion:r}={serializeVersion:!0}){D.useEffect(()=>{const i=window.parent,o=Mf(t,{serializeVersion:r}).toString(),s=new URL(i.location.toString());if(!s.search){s.search=o,i.history.replaceState(null,"",s);return}s.search.slice(1)!==o&&(s.search=o,i.history.pushState(null,"",s))}),parent.onpopstate=()=>{const i=window.parent,o=new URLSearchParams(i.location.search),s=$f(o);s&&(e.setState(s),n(e.getState()))}}const us=function(e){let t,n=!1;return function(...r){return n||(t=e(r),n=!0),t}};function Oy(e){var t,n;return typeof((n=(t=e.wap_tms)==null?void 0:t.custom)==null?void 0:n.trackComponentClick)!="function"?null:e.wap_tms.custom.trackComponentClick.bind(e.wap_tms.custom)}class _y{constructor(){Ve(this,"_window");Ve(this,"_consoleNotification",{notInitialized:us(()=>console.log("Adobe analytics is not initialized")),notFound:us(()=>console.log("Adobe analytics not found on a page")),devMode:us(()=>console.log("Analytics in dev mode"))});Ve(this,"_send",t=>{if(!this._window){this._consoleNotification.notInitialized();return}const n=Mf(Tn.getState()).toString(),r=Oy(this._window);if(!r){this._consoleNotification.notFound();return}try{r(t,n)}catch(i){console.error(i)}})}initialize(t){this._window=t}install(){this._send("install")}combinationView(){this._send("combination-view")}}const Ge=new _y;function Ny(){const e=Sy();e&&Tn.setState(e);const t=D.createContext((r,i)=>{Tn.select(r,i)});function n(){const[r,i]=D.useState(Tn.getState());return xy(Tn,r,i),[r,(o,s)=>i(Tn.select(o,s))]}return{SelectorContext:t,useSelector:n}}async function Py(e){e&&(navigator.clipboard?await navigator.clipboard.writeText(e):Ey(e))}function Ey(e){const t=Cy(e);document.body.append(t),t.select(),document.execCommand("copy"),t.remove()}function Cy(e){const t=document.createElement("textarea");t.style.fontSize="12pt",t.style.border="0",t.style.padding="0",t.style.margin="0",t.style.position="absolute",t.style.left="-9999px";const n=window.pageYOffset||document.documentElement.scrollTop;return t.style.top=`${n}px`,t.setAttribute("readonly",""),t.value=e,t}function jy(){return m.jsxs("svg",{version:"1.1",width:"24",height:"24",viewBox:"0 0 205 205",xmlns:"http://www.w3.org/2000/svg",className:"svg-icon",children:[m.jsx("path",{fill:"none",stroke:"currentColor",strokeWidth:"10",d:"M 50 145 a 15 15 0 0 1 -15 -15 v -90 a 15 15 0 0 1 15 -15 h 70 a 15 15 0 0 1 15 15 v 5"}),m.jsx("rect",{x:"65",y:"60",width:"100",height:"120",rx:"15",fill:"none",stroke:"currentColor",strokeWidth:"10"})]})}function Iy(){return m.jsx("svg",{version:"1.1",width:"24",height:"24",viewBox:"0 0 200 200",xmlns:"http://www.w3.org/2000/svg",className:"svg-icon",children:m.jsx("path",{strokeLinejoin:"round",strokeLinecap:"round",fill:"none",stroke:"currentColor",strokeWidth:"15",d:"M 40 100 L 90 150 L 170 40"})})}function Ry(){return m.jsxs("svg",{version:"1.1",width:"24",height:"24",viewBox:"0 0 200 200",xmlns:"http://www.w3.org/2000/svg",className:"svg-icon",children:[m.jsx("circle",{cx:"100",cy:"100",r:"90",fill:"none",strokeWidth:"15",stroke:"currentColor"}),m.jsx("path",{fill:"none",stroke:"currentColor",strokeWidth:"15",strokeLinecap:"round",d:"M 65 80 A 35 35 0 1 1 100 115"}),m.jsx("circle",{cx:"100",cy:"150",r:"3",fill:"none",stroke:"currentColor",strokeWidth:"15"})]})}const b=({comment:e,command:t,onCopy:n})=>{const[r,i]=D.useState(!1),o=async()=>{r||(await Py(t),i(!0),setTimeout(()=>i(!1),1500),n==null||n())};return m.jsxs("div",{className:"st-code-snippet","data-cy":"instructions-step",children:[e&&m.jsx(Bf,{children:e}),m.jsxs("div",{"data-cy":"command",children:[t&&m.jsx("code",{className:"st-code-snippet-content",children:t}),t&&m.jsx("button",{className:"copy-button",type:"button","aria-label":"Copy","data-cy":"copy",onClick:o,children:r?m.jsx(Iy,{}):m.jsx(jy,{})})]})]})},Bf=({children:e})=>m.jsxs("pre",{className:"st-code-snippet-comment",children:["# ",e]}),Ty=({comment:e,snippets:t})=>m.jsxs("div",{className:"st-code-snippet-multi-line","data-cy":"command",children:[e&&m.jsx(Bf,{children:e}),t.map(n=>m.jsx(b,{...n},n.command))]});function Ly(e){return e.host==="docs.openvino.ai"}const cs="production.docs.en",Kf=(cs==null?void 0:cs.includes("idz"))||!1,ds={link:"spark-hyperlink spark-hyperlink-primary spark-hyperlink-standard spark-focus-visible spark-focus-visible-self spark-focus-visible-snap spark-focus-visible-background",button:"spark-button spark-button-action spark-button-size-m spark-focus-visible spark-focus-visible-self spark-focus-visible-snap",buttonContent:"spark-button-content"},Oe=({href:e,children:t,type:n="link",testId:r="link",onClick:i})=>{const o=!Kf&&Ly(new URL(e))?"_parent":"_blank";return n==="link"?m.jsx("a",{href:e,className:ds.link,target:o,rel:"noreferrer noopener","data-cy":r,onClick:()=>i==null?void 0:i(),children:t}):m.jsx("span",{className:ds.button,children:m.jsx("span",{className:ds.buttonContent,children:m.jsx("a",{href:e,target:o,rel:"noreferrer noopener","data-cy":r,onClick:()=>i==null?void 0:i(),children:t})})})},Ay={heading:"spark-heading spark-font-200"},Ce=({title:e,accent:t=!1,dashed:n=!1,children:r,testId:i})=>m.jsxs("div",{className:`st-section ${t?"st-section-accent":""} ${n?"st-section-dashed":""}`,"data-cy":i,children:[m.jsx("span",{className:`st-section-title ${Ay.heading}`,children:e}),m.jsx("div",{className:"st-section-content",children:D.Children.map(r,o=>m.jsx(Fy,{children:o}))})]}),Fy=({children:e})=>m.jsx("div",{className:"st-section-content-row",children:e}),Dy=({version:e,distribution:t})=>{const{t:n}=$("translation",{keyPrefix:"distributions.apt.steps"}),{t:r}=$("translation",{keyPrefix:"selectorForm.titles"}),{commands:i}=t.data,o={comment:m.jsxs(z,{ns:"translation",i18nKey:"distributions.apt.steps.addRepository",children:[m.jsx("b",{children:"Step 3:"})," Add the repository via the following command"]}),snippets:i.getAddRepositoryCommands(e,t.data.os).map(({ubuntuVersionNumber:l,command:a})=>({comment:`Ubuntu ${l}`,command:a}))},s={downloadKey:{comment:m.jsxs(z,{t:n,i18nKey:"download",values:{filename:i.keyFilename},children:[m.jsx("b",{children:"Step 1:"})," Download the ",m.jsx(Oe,{href:i.keyHref,children:i.keyFilename}),". You can also use the following command"]}),command:i.downloadKeyCommand},addKey:{comment:m.jsxs(z,{t:n,i18nKey:"addKey",children:[m.jsx("b",{children:"Step 2:"})," Add this key to the system keyring"]}),command:i.addKeyCommand},addRepository:o,updatePackages:{comment:m.jsxs(z,{t:n,i18nKey:"updateList",children:[m.jsx("b",{children:"Step 4:"})," Update the list of packages via the update command"]}),command:i.updatePackagesCommand},verifyAptCache:{comment:m.jsxs(z,{t:n,i18nKey:"verify",children:[m.jsx("b",{children:"Step 5:"})," Verify that the APT repository is properly set up. Run the apt-cache command to see a list of all available OpenVINO packages and components"]}),command:i.verifyAptCacheCommand},install:{comment:m.jsxs(z,{t:n,i18nKey:"install",children:[m.jsx("b",{children:"Step 6:"})," Install OpenVINO Runtime"]}),command:i.getInstallCommand(e),onCopy:()=>Ge.install()}};return m.jsxs(Ce,{title:r("install"),accent:!0,testId:"instructions",children:[m.jsx(b,{...s.downloadKey}),m.jsx(b,{...s.addKey}),m.jsx(Ty,{...s.addRepository}),m.jsx(b,{...s.updatePackages}),m.jsx(b,{...s.verifyAptCache}),m.jsx(b,{...s.install})]})},Uy=({distribution:e})=>{const{t}=$("translation",{keyPrefix:"distributions.brew.steps"}),{t:n}=$("translation",{keyPrefix:"selectorForm.titles"}),{commands:r}=e.data,i={install:{comment:m.jsx(z,{t,i18nKey:"install",children:"Download and install the package"}),command:r.install,onCopy:()=>Ge.install()}};return m.jsx(Ce,{title:n("install"),accent:!0,testId:"instructions",children:m.jsx(b,{...i.install})})},zy=({version:e,distribution:t})=>{const{t:n}=$("translation",{keyPrefix:"distributions.conan.steps"}),{t:r}=$("translation",{keyPrefix:"selectorForm.titles"}),{commands:i}=t.data,{txtFilename:o,cmakeFilename:s}=i,l={createConanFile:{comment:m.jsxs(z,{t:n,i18nKey:"createConanFile",values:{txtFilename:o},children:[m.jsx("b",{children:"Step 1:"})," Create a ",m.jsx("b",{children:o})," file for your OpenVINO project and add “openvino” dependency in there"]}),command:i.conanTXTContent(e)},install:{comment:m.jsxs(z,{t:n,i18nKey:"install",values:{cmakeFilename:s},children:[m.jsx("b",{children:"Step 2:"})," Run the command below to create ",m.jsx("b",{children:s})," file, which will be used to compile your project with OpenVINO"]}),command:i.install,onCopy:()=>Ge.install()},compile:{comment:m.jsxs(z,{t:n,i18nKey:"compile",children:[m.jsx("b",{children:"Step 3:"})," Configure and compile your project with OpenVINO"]}),command:i.compile}};return m.jsxs(Ce,{title:r("install"),accent:!0,testId:"instructions",children:[m.jsx(b,{...l.createConanFile}),m.jsx(b,{...l.install}),m.jsx(b,{...l.compile})]})},Vy=({version:e,distribution:t})=>{const{t:n}=$("translation",{keyPrefix:"distributions.conda.steps"}),{t:r}=$("translation",{keyPrefix:"selectorForm.titles"}),{commands:i}=t.data,o={createEnv:{comment:m.jsxs(z,{t:n,i18nKey:"createEnv",children:[m.jsx("b",{children:"Step 1:"})," Create the Anaconda environment (Python 3.10 used as an example)"]}),command:i.createEnv},activateEnv:{comment:m.jsxs(z,{t:n,i18nKey:"activateEnv",children:[m.jsx("b",{children:"Step 2:"})," Activate the Anaconda environment"]}),command:i.activateEnv},upgradePip:{comment:m.jsxs(z,{t:n,i18nKey:"update",children:[m.jsx("b",{children:"Step 3:"})," Update the Anaconda to latest version"]}),command:i.update},install:{comment:m.jsxs(z,{t:n,i18nKey:"install",children:[m.jsx("b",{children:"Step 4:"})," Download and install the package"]}),command:i.getInstall(e),onCopy:()=>Ge.install()}};return m.jsxs(Ce,{title:r("install"),accent:!0,testId:"instructions",children:[m.jsx(b,{...o.createEnv}),m.jsx(b,{...o.activateEnv}),m.jsx(b,{...o.upgradePip}),m.jsx(b,{...o.install})]})},fs=({ovPackage:e,distribution:t})=>{const{t:n}=$("translation",{keyPrefix:"distributions.download"}),{t:r}=$("translation",{keyPrefix:"selectorForm.titles"}),i={[A.ARCHIVE]:e.key===Ne.OPENVINO_BASE?n("downloadArchives"):n("downloadArchivesGenAI"),[A.DOCKER]:n("gotoDocker"),[A.SNAP]:n("gotoInstallInstruction")}[t.key],o=m.jsxs(m.Fragment,{children:[n("useFollowingLink"),m.jsx("br",{}),m.jsx("b",{children:m.jsx(Oe,{href:t.data.downloadLink,testId:"download-button",onClick:()=>Ge.install(),children:i})})]});return m.jsx(Ce,{title:r("install"),accent:!0,testId:"instructions",children:m.jsx(b,{comment:o})})},by=({ovPackage:e,version:t,distribution:n})=>{const{t:r}=$("translation",{keyPrefix:"distributions.githubGitee"}),{t:i}=$("translation",{keyPrefix:"selectorForm.titles"}),o={clone:{comment:m.jsxs(z,{t:r,i18nKey:"steps.useGitClone",children:[m.jsx("b",{children:"Step 1:"})," Use Git to clone the OpenVINO toolkit repository"]}),command:n.data.commands.getCloneCommand(e,t),onCopy:()=>Ge.install()},build:{comment:m.jsxs(z,{t:r,i18nKey:"steps.buildInstructions",children:[m.jsx("b",{children:"Step 2:"})," Follow the ",m.jsx(Oe,{href:n.data.links.getBuildInstructionsLink(e,t),testId:"build-instructions-link",children:"instructions to build from source"})]})}};return m.jsxs(Ce,{title:i("install"),accent:!0,testId:"instructions",children:[m.jsx(b,{...o.clone}),m.jsx(b,{...o.build})]})},My=({distribution:e,version:t})=>{const{t:n}=$("translation",{keyPrefix:"distributions.npm.steps"}),{t:r}=$("translation",{keyPrefix:"selectorForm.titles"}),{commands:i}=e.data,o={install:{comment:m.jsx(z,{t:n,i18nKey:"install",children:"Download and install the package"}),command:i.getInstall(t),onCopy:()=>Ge.install()}};return m.jsx(Ce,{title:r("install"),accent:!0,testId:"instructions",children:m.jsx(b,{...o.install})})},$y=({ovPackage:e,environment:t,os:n,version:r,distribution:i})=>{const{t:o}=$("translation",{keyPrefix:"distributions.pip.steps"}),{t:s}=$("translation",{keyPrefix:"selectorForm.titles"}),{commands:l}=i.data,a=l.getCreateVenvCommand(n,r),u=l.getActivateVenvCommand(n,r),p=l.getInstallCommand({ovPackage:e,environment:t,os:n,version:r,distribution:i}),d={createEnv:{comment:m.jsxs(z,{t:o,i18nKey:"createVenv",children:[m.jsx("b",{children:"Step 1:"})," Create virtual environment"]}),command:a},activateEnv:{comment:m.jsxs(z,{t:o,i18nKey:"activateVenv",children:[m.jsx("b",{children:"Step 2:"})," Activate virtual environment"]}),command:u},upgradePip:{comment:m.jsxs(z,{t:o,i18nKey:"upgradePip",children:[m.jsx("b",{children:"Step 3:"})," Upgrade pip to latest version"]}),command:l.upgradeCommand},install:{comment:m.jsxs(z,{t:o,i18nKey:"install",children:[m.jsx("b",{children:"Step 4:"})," Download and install the package"]}),command:p,onCopy:()=>Ge.install()}};return m.jsxs(Ce,{title:s("install"),accent:!0,testId:"instructions",children:[m.jsx(b,{...d.createEnv}),m.jsx(b,{...d.activateEnv}),m.jsx(b,{...d.upgradePip}),m.jsx(b,{...d.install})]})},By=({distribution:e})=>{const{t}=$("translation",{keyPrefix:"distributions.vcpkg.steps"}),{t:n}=$("translation",{keyPrefix:"selectorForm.titles"}),{commands:r}=e.data,i={install:{comment:m.jsx(z,{t,i18nKey:"install",children:"Download and install the package"}),command:r.install,onCopy:()=>Ge.install()}};return m.jsx(Ce,{title:n("install"),accent:!0,testId:"instructions",children:m.jsx(b,{...i.install})})},Ky=({version:e,distribution:t})=>{const{t:n}=$("translation",{keyPrefix:"distributions.yum.steps"}),{t:r}=$("translation",{keyPrefix:"selectorForm.titles"}),{yumYear:i}=e.metadata,{commands:o}=t.data,s={createRepo:{comment:m.jsxs(z,{t:n,i18nKey:"createRepoFile",children:[m.jsx("b",{children:"Step 1:"})," Create the YUM repo file in the /tmp directory as a normal user"]}),command:o.getCreateRepoCommand(e)},moveRepoFile:{comment:m.jsxs(z,{t:n,i18nKey:"moveFile",values:{year:i,directory:o.directory},children:[m.jsx("b",{children:"Step 2:"})," Move the new openvino-",{year:i},".repo file to the YUM configuration directory ",m.jsx("b",{children:o.directory})]}),command:o.getMoveRepoFileCommand(e)},verifyRepo:{comment:m.jsxs(z,{t:n,i18nKey:"verify",children:[m.jsx("b",{children:"Step 3:"})," Verify that the new repo is properly setup by running the following command"]}),command:o.verifyRepoCommand},install:{comment:m.jsxs(z,{t:n,i18nKey:"install",children:[m.jsx("b",{children:"Step 4:"})," Install OpenVINO Runtime"]}),command:o.getInstallCommand(e),onCopy:()=>Ge.install()}};return m.jsxs(Ce,{title:r("install"),accent:!0,testId:"instructions",children:[m.jsx(b,{...s.createRepo}),m.jsx(b,{...s.moveRepoFile}),m.jsx(b,{...s.verifyRepo}),m.jsx(b,{...s.install})]})},Hy=({version:e,distribution:t})=>{const{t:n}=$("translation",{keyPrefix:"distributions.zypper.steps"}),{t:r}=$("translation",{keyPrefix:"selectorForm.titles"}),{commands:i}=t.data,o={addRepo:{comment:m.jsxs(z,{t:n,i18nKey:"addRepo",children:[m.jsx("b",{children:"Step 1:"})," Create a ZYPPER repository file with the command below"]}),command:i.addRepo},refresh:{comment:m.jsxs(z,{t:n,i18nKey:"refresh",children:[m.jsx("b",{children:"Step 2:"})," Refresh repositories"]}),command:i.refresh},install:{comment:m.jsxs(z,{t:n,i18nKey:"install",children:[m.jsx("b",{children:"Step 3:"})," Install OpenVINO"]}),command:i.getInstallCommand(e),onCopy:()=>Ge.install()}};return m.jsxs(Ce,{title:r("install"),accent:!0,testId:"instructions",children:[m.jsx(b,{...o.addRepo}),m.jsx(b,{...o.refresh}),m.jsx(b,{...o.install})]})},Wy=({state:e})=>{var r;const t={ovPackage:e.PACKAGE.selected,environment:(r=e.ENVIRONMENT)==null?void 0:r.selected,os:e.OP_SYSTEM.selected,version:e.VERSION.selected,distribution:e.DISTRIBUTION.selected};if(t.distribution.key===A.PIP)return m.jsx($y,{...t,distribution:t.distribution});if(t.distribution.key===A.ARCHIVE)return m.jsx(fs,{...t,distribution:t.distribution});if(t.distribution.key===A.DOCKER)return m.jsx(fs,{...t,distribution:t.distribution});if(t.distribution.key===A.GITHUB||t.distribution.key===A.GITEE)return m.jsx(by,{...t,distribution:t.distribution});if(t.distribution.key===A.APT)return m.jsx(Dy,{...t,distribution:t.distribution});if(t.distribution.key===A.YUM)return m.jsx(Ky,{...t,distribution:t.distribution});if(t.distribution.key===A.CONDA)return m.jsx(Vy,{...t,distribution:t.distribution});if(t.distribution.key===A.BREW)return m.jsx(Uy,{...t,distribution:t.distribution});if(t.distribution.key===A.VCPKG)return m.jsx(By,{...t,distribution:t.distribution});if(t.distribution.key===A.CONAN)return m.jsx(zy,{...t,distribution:t.distribution});if(t.distribution.key===A.NPM)return m.jsx(My,{...t,distribution:t.distribution});if(t.distribution.key===A.ZYPPER)return m.jsx(Hy,{...t,distribution:t.distribution});if(t.distribution.key===A.SNAP)return m.jsx(fs,{...t,distribution:t.distribution});const n=t.distribution;throw new Error(`${n}`)};function Gy(){const{t:e}=$("common",{keyPrefix:"relatedTools"}),{t}=$("translation");return m.jsx(Ce,{title:t("selectorForm.titles.relatedTools"),testId:"relatedTools",accent:!0,dashed:!0,children:m.jsxs("div",{className:"st-related-tools-links",children:[m.jsx(Oe,{href:"https://github.com/openvinotoolkit/openvino_notebooks",testId:"notebooks-link",children:e("OpenVINONotebooks")}),m.jsx(Oe,{href:"https://huggingface.co/docs/optimum/main/intel/openvino/inference",testId:"hf_optimum-link",children:"Hugging Face + Optimum Intel"}),m.jsx("div",{children:m.jsxs(z,{t:e,i18nKey:"tokenizers",children:[m.jsx(Oe,{href:"https://docs.openvino.ai/2024/learn-openvino/llm_inference_guide/ov-tokenizers.html",testId:"openvino_tokenizers-link",children:"OpenVINO Tokenizers"}),"to streamline tokenizer conversion"]})}),m.jsx("div",{children:m.jsxs(z,{t:e,i18nKey:"nncf",children:[m.jsx(Oe,{href:"https://docs.openvino.ai/2024/openvino-workflow/model-optimization-guide/compressing-models-during-training.html",testId:"nncf-link",children:"NNCF"}),"for implementing compression algorithms on models"]})}),m.jsx("div",{children:m.jsxs(z,{t:e,i18nKey:"ovms",children:[m.jsx(Oe,{href:"https://docs.openvino.ai/2024/ovms_what_is_openvino_model_server.html",testId:"ovms-link",children:"OVMS"}),"for serving models optimized for deployment"]})})]})})}function Yy({state:e}){const t=e.PACKAGE.selected,n=e.DISTRIBUTION.selected,r=e.VERSION.selected,{t:i}=$("translation",{keyPrefix:"selectorForm.titles"}),{t:o}=$("common",{keyPrefix:"resources"});let s=m.jsx(m.Fragment,{});if(A.GITHUB===n.key||A.GITEE===n.key){const l=n.key===A.GITHUB?t.key===Ne.OPENVINO_BASE?o("githubRepository"):o("githubGenAIRepository"):t.key===Ne.OPENVINO_BASE?o("giteeRepository"):o("giteeGenAIRepository");s=m.jsxs(m.Fragment,{children:[m.jsx(Oe,{href:n.data.links.getBuildInstructionsLink(t,r),testId:"install-instructions-link",children:o("installationInstructions")}),m.jsx(Oe,{href:n.data.links.getRepositoryLink(t,r),testId:"repository-link",children:l})]})}else s=m.jsx(Oe,{href:n.data.linksSet.installation,testId:"install-instructions-link",children:o("installationInstructions")});return m.jsx(Ce,{title:i("resources"),testId:"resources",accent:!0,children:m.jsxs("div",{className:"st-resources-links",children:[m.jsxs("div",{children:[s,m.jsx(Oe,{href:"https://github.com/openvinotoolkit/openvino/releases",testId:"previous-releases-link",children:o("prevReleases")}),m.jsx(Oe,{href:r.metadata.systemRequirementsLink,testId:"system-requirements-link",children:o("systemRequirements")})]}),m.jsxs("div",{children:[m.jsx(Oe,{href:r.metadata.getStartedLink,testId:"get-started-link",children:o("getStarted")}),m.jsx(Oe,{href:r.metadata.troubleshootingLink,testId:"troubleshooting-link",children:o("troubleshooting")})]})]})})}const Oi={tooltipToggle:"spark-tooltip-toggle",tooltipPopover:"spark-tooltip spark-tooltip-size-m spark-shadow spark-tooltip-bottom",tooltipLabel:"spark-tooltip-label",tooltipArrow:"spark-tooltip-tip"},Qy=({content:e})=>{const[t,n]=D.useState(!1);return m.jsx("div",{className:"help-tooltip",children:m.jsxs("div",{className:Oi.tooltipToggle,onMouseEnter:()=>n(!0),onMouseLeave:()=>n(!1),children:[m.jsx("div",{"aria-hidden":"true",role:"img",className:"help-tooltip-icon",children:m.jsx(Ry,{})}),t&&m.jsxs("div",{className:`help-tooltip-popover ${Oi.tooltipPopover}`,role:"tooltip",children:[m.jsx("span",{className:`help-tooltip-popover-content ${Oi.tooltipLabel}`,children:e}),m.jsx("span",{className:Oi.tooltipArrow})]})]})})},ps={toggleSwitchField:"spark-fieldlabel spark-fieldlabel-size-m spark-toggle-switch spark-toggle-switch-action spark-toggle-switch-size-l",toggleSwitchInput:"spark-focus-visible spark-focus-visible-suppress spark-focus-visible-adjacent",toggleSwitchSelector:"spark-toggle-switch-selector spark-focus-visible spark-focus-visible-snap"},Xy=({label:e,checked:t=!1,onClick:n,testId:r})=>m.jsxs("label",{className:ps.toggleSwitchField,"data-cy":r,children:[m.jsx("input",{type:"checkbox",className:ps.toggleSwitchInput,role:"switch",checked:t,onChange:()=>n==null?void 0:n()}),m.jsx("span",{className:ps.toggleSwitchSelector}),m.jsx("div",{children:e})]}),cn={toggleButton:"spark-button spark-button-size-l spark-focus-visible spark-focus-visible-self spark-focus-visible-snap",toggleButtonGroup:"spark-button-group spark-button-group-orientation-horizontal spark-button-group-align-start spark-button-group-spacing-l",actionButton:"spark-button-action",secondaryButton:"spark-button-secondary",disabledButton:"spark-button-disabled",buttonContent:"spark-button-content",fontXs:"spark-font-25"},Hf=({onClick:e,checked:t=!1,disabled:n=!1,title:r,subtitle:i,value:o})=>m.jsx("button",{className:`${cn.toggleButton} ${t?cn.actionButton:cn.secondaryButton} ${n&&cn.disabledButton}`,type:"button",role:"radio","aria-checked":t,onClick:()=>e==null?void 0:e(),"data-cy":o,"aria-label":r,children:m.jsxs("span",{className:cn.buttonContent,children:[m.jsx("span",{className:"title",children:r}),i&&m.jsx("span",{className:`${cn.fontXs} subtitle`,children:i})]})}),Wf=({children:e,className:t})=>m.jsx("div",{className:`option-button-group ${t||""} ${cn.toggleButtonGroup}`,children:e});function Jy({title:e,options:t,level:n}){const r=D.useContext(ma),i=t.some(({checked:u})=>u),[o,s]=D.useState(i),l=t.map(({level:u,key:p,checked:d,metadata:h})=>m.jsx(Hf,{value:`${u}_${p}`,checked:d,title:h.title,onClick:()=>r(u,p)},p)),{t:a}=$("translation",{keyPrefix:"frameworks"});return m.jsxs(Ce,{title:e,testId:n,children:[m.jsxs(m.Fragment,{children:[m.jsx(Xy,{label:a("install"),checked:o,onClick:()=>s(!o),testId:"frameworks-toggle-switch"}),m.jsx(Qy,{content:a("tooltip")})]}),m.jsx(m.Fragment,{children:o&&m.jsx(Wf,{children:l})})]})}function wr({title:e,options:t,level:n}){const r=D.useContext(ma),i=t.map(({level:o,key:s,checked:l,metadata:a})=>m.jsx(Hf,{value:`${o}_${s}`,checked:l,title:a.title,subtitle:a.subtitle,onClick:()=>r(o,s)},s));return m.jsx(Ce,{title:e,testId:n,children:m.jsx(Wf,{children:i})})}function Zy({state:e}){var a,u;const t=e.PACKAGE.nodes,n=e.VERSION.nodes,r=(a=e.ENVIRONMENT)==null?void 0:a.nodes.map(p=>({...p,metadata:{...p.metadata,subtitle:void 0}})),i=e.OP_SYSTEM.nodes,o=e.DISTRIBUTION.nodes,s=(u=e.FRAMEWORK)==null?void 0:u.nodes;D.useEffect(()=>Ge.combinationView(),[e]);const{t:l}=$("translation",{keyPrefix:"selectorForm.titles"});return m.jsxs(m.Fragment,{children:[m.jsx(wr,{title:l("package"),options:t,level:j.PACKAGE}),m.jsx(wr,{title:l("version"),options:n,level:j.VERSION}),r&&m.jsx(wr,{title:l("envinronment"),options:r,level:j.ENVIRONMENT}),m.jsx(wr,{title:l("os"),options:i,level:j.OP_SYSTEM}),m.jsx(wr,{title:l("distribution"),options:o,level:j.DISTRIBUTION}),s&&m.jsx(Jy,{title:l("frameworks"),options:s,level:j.FRAMEWORK})]})}const{SelectorContext:ma,useSelector:qy}=Ny();Ge.initialize(window.parent);function e0(){const[e,t]=qy();return m.jsx("div",{className:`st-responsive-container ${Kf?"idz-page":""}`,children:m.jsxs(ma.Provider,{value:t,children:[m.jsx(Zy,{state:e}),m.jsx(Wy,{state:e}),m.jsx(Yy,{state:e}),m.jsx(Gy,{})]})})}vs.createRoot(document.getElementById("root")).render(m.jsx(cp.StrictMode,{children:m.jsx(e0,{})})); diff --git a/docs/sphinx_setup/_static/selector-tool/selector-451bede.html b/docs/sphinx_setup/_static/selector-tool/selector-2a63478.html similarity index 78% rename from docs/sphinx_setup/_static/selector-tool/selector-451bede.html rename to docs/sphinx_setup/_static/selector-tool/selector-2a63478.html index c64aef27138b60..6bff085dfdb3be 100644 --- a/docs/sphinx_setup/_static/selector-tool/selector-451bede.html +++ b/docs/sphinx_setup/_static/selector-tool/selector-2a63478.html @@ -1,7 +1,7 @@ - + Download Intel® Distribution of OpenVINO™ Toolkit @@ -9,7 +9,7 @@ name="description" content="Download a version of the Intel® Distribution of OpenVINO™ toolkit for Linux, Windows, or macOS." /> - + diff --git a/docs/sphinx_setup/index.rst b/docs/sphinx_setup/index.rst index 2e6f960468015f..4da0aa8f29535c 100644 --- a/docs/sphinx_setup/index.rst +++ b/docs/sphinx_setup/index.rst @@ -11,8 +11,8 @@ generative AI, video, audio, and language with models from popular frameworks li TensorFlow, ONNX, and more. Convert and optimize models, and deploy across a mix of Intel® hardware and environments, on-premises and on-device, in the browser or in the cloud. -Check out the `OpenVINO Cheat Sheet. `__ - +Check out the `OpenVINO Cheat Sheet [PDF] `__ +Check out the `GenAI Quick-start Guide [PDF] `__ .. container:: diff --git a/samples/js/node/fetch-samples-assets.js b/samples/js/node/fetch-samples-assets.js index 33dd509a922f85..c75d913d74c93e 100644 --- a/samples/js/node/fetch-samples-assets.js +++ b/samples/js/node/fetch-samples-assets.js @@ -28,6 +28,7 @@ const models = [ // question answering '/repositories/open_model_zoo/2022.3/models_bin/1/bert-small-uncased-whole-word-masking-squad-0001/FP16/bert-small-uncased-whole-word-masking-squad-0001.xml', + '/repositories/open_model_zoo/2022.3/models_bin/1/bert-small-uncased-whole-word-masking-squad-0001/FP16/bert-small-uncased-whole-word-masking-squad-0001.bin', ]; const modelsDir = __dirname + '/../assets/models'; diff --git a/samples/js/node/package-lock.json b/samples/js/node/package-lock.json index 96a013fb0435c7..020cec71ea3103 100644 --- a/samples/js/node/package-lock.json +++ b/samples/js/node/package-lock.json @@ -15,7 +15,7 @@ "args": "^5.0.3", "eslint": "^8.39.0", "https-proxy-agent": "^7.0.2", - "openvino-node": "^2024.4.0" + "openvino-node": "^2024.5.0-0" }, "engines": { "node": ">=21.0.0" @@ -194,108 +194,6 @@ "@napi-rs/canvas-win32-x64-msvc": "0.1.59" } }, - "node_modules/@napi-rs/canvas-android-arm64": { - "version": "0.1.59", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.59.tgz", - "integrity": "sha512-p4rRL9KIDz57Z+gKLpemX36DB7fVVHmY4DtesMGrnjx4gSBUM2M7LNzbzf4o3oPZGDiHMY0vnvNHR4dKfszNeg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/canvas-darwin-arm64": { - "version": "0.1.59", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.59.tgz", - "integrity": "sha512-+8s06WxcM9ilv9PVOl57hvasbwKWMfrrNAYknqMPCn4jpc4XDcLbrM5LTZGhhptlv9jQ9DmHfZ978/xInsMYXw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/canvas-darwin-x64": { - "version": "0.1.59", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.59.tgz", - "integrity": "sha512-6kziJHjXdxduYK2L2uuwjEIYoPJednKq+C81MCm3fPobXE4HBKs0JGXwq3GkWNe340U340vmagwXiFi6muEy+g==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": { - "version": "0.1.59", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.59.tgz", - "integrity": "sha512-eCkyS7jojNmaUPaVFdNjAyS0R3isrJtUfRf1vRP6K50GRuHso3vwQRbZBPKM71qHdjPDylfaQc5H6/M7epyD+w==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/canvas-linux-arm64-gnu": { - "version": "0.1.59", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.59.tgz", - "integrity": "sha512-1u4++lbsolP1MAPViuDoZmgmDLKlV0iJnlHN2dfwgbu3t53P0l3jIT1oCIAiWil0OlrWtDF24JbY7LUUGH5aHg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/canvas-linux-arm64-musl": { - "version": "0.1.59", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.59.tgz", - "integrity": "sha512-eqevZ2kWPxeAnvhxl7U5tf6AiMnhlO4w2Hci79WQkfeirqQG6RRM4Jnxbh9iO3jkAnnOXmM4r+S3UrOcfIx1Rg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, "node_modules/@napi-rs/canvas-linux-x64-gnu": { "version": "0.1.59", "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.59.tgz", @@ -330,23 +228,6 @@ "node": ">= 10" } }, - "node_modules/@napi-rs/canvas-win32-x64-msvc": { - "version": "0.1.59", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.59.tgz", - "integrity": "sha512-bYMiZJsKPkU7HEoYI5E0alOSV1EkaigY4VEgGHPK9W/qGMmNFsxdbURQqa5h3zbhZTK5QRSdYYqowcTEYVIlug==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -1053,10 +934,11 @@ "dev": true }, "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dev": true, + "license": "MIT", "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -2038,11 +1920,12 @@ } }, "node_modules/openvino-node": { - "version": "2024.4.0", - "resolved": "https://registry.npmjs.org/openvino-node/-/openvino-node-2024.4.0.tgz", - "integrity": "sha512-yEix2i3gXeO77XcA6x6TJm9c8HDrZHnW6tSiYcbanMYVpeDZ0E5jNikxoiMmN7dVR4E6t+VWr05eexOROqrpkw==", + "version": "2024.5.0-0", + "resolved": "https://registry.npmjs.org/openvino-node/-/openvino-node-2024.5.0-0.tgz", + "integrity": "sha512-SgvHH3OdOXyMu5iZx0oBFWn7yIu3uB54IIfmXFKlyhHbSjO+3ph+DauUdlUkp2DGETR7bzq7+cPyyroeOF7qqQ==", "dev": true, "hasInstallScript": true, + "license": "Apache-2.0", "os": [ "win32", "darwin", diff --git a/samples/js/node/package.json b/samples/js/node/package.json index d392a72143de03..b3e12a265f0c77 100644 --- a/samples/js/node/package.json +++ b/samples/js/node/package.json @@ -8,7 +8,7 @@ "args": "^5.0.3", "eslint": "^8.39.0", "https-proxy-agent": "^7.0.2", - "openvino-node": "^2024.4.0", + "openvino-node": "^2024.5.0-0", "@napi-rs/canvas": "^0.1.59" }, "scripts": { diff --git a/src/bindings/js/node/.npmignore b/src/bindings/js/node/.npmignore index e4fe3743e7de30..0d52e466ac2d8f 100644 --- a/src/bindings/js/node/.npmignore +++ b/src/bindings/js/node/.npmignore @@ -1,6 +1,7 @@ bin include -lib +/lib +!/scripts/lib src tests thirdparty diff --git a/src/bindings/js/node/package-lock.json b/src/bindings/js/node/package-lock.json index 41a813579ad83c..27f426968e5b54 100644 --- a/src/bindings/js/node/package-lock.json +++ b/src/bindings/js/node/package-lock.json @@ -1,12 +1,12 @@ { "name": "openvino-node", - "version": "2024.4.0", + "version": "2024.5.0-0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "openvino-node", - "version": "2024.4.0", + "version": "2024.5.0-0", "hasInstallScript": true, "license": "Apache-2.0", "os": [ @@ -32,44 +32,41 @@ "node": ">=21.0.0" } }, - "node_modules/@aashutoshrathi/word-wrap": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz", - "integrity": "sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/@eslint-community/eslint-utils": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", - "integrity": "sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==", + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.1.tgz", + "integrity": "sha512-s3O3waFUrMV8P/XaF/+ZTp1X9XBZW1a4B97ZnjQF2KYWaFD2A8KyFBsrsfSjEmjn3RGWAIuvlneuZm3CUK3jbA==", "dev": true, + "license": "MIT", "dependencies": { - "eslint-visitor-keys": "^3.3.0" + "eslint-visitor-keys": "^3.4.3" }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, + "funding": { + "url": "https://opencollective.com/eslint" + }, "peerDependencies": { "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" } }, "node_modules/@eslint-community/regexpp": { - "version": "4.8.1", - "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.8.1.tgz", - "integrity": "sha512-PWiOzLIUAjN/w5K17PoF4n6sKBw0gqLHPhywmYHP4t1VFQQVYeb1yWsJwnMVEMl3tUHME7X/SJPZLmtG7XBDxQ==", + "version": "4.12.1", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.1.tgz", + "integrity": "sha512-CCZCDJuduB9OUkFkY2IgppNZMi2lBQgD2qzwXkEia16cge2pijY/aXi96CJMquDMn3nJdlPV1A5KrJEXwfLNzQ==", "dev": true, + "license": "MIT", "engines": { "node": "^12.0.0 || ^14.0.0 || >=16.0.0" } }, "node_modules/@eslint/eslintrc": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.2.tgz", - "integrity": "sha512-+wvgpDsrB1YqAMdEUCcnTlpfVBH7Vqn6A/NT3D8WVXFIaKMlErPIZT3oCIAVCOtarRpMtelZLqJeU3t7WY6X6g==", + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.4.tgz", + "integrity": "sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==", "dev": true, + "license": "MIT", "dependencies": { "ajv": "^6.12.4", "debug": "^4.3.2", @@ -88,34 +85,86 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/@eslint/eslintrc/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/@eslint/eslintrc/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/@eslint/js": { - "version": "8.49.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.49.0.tgz", - "integrity": "sha512-1S8uAY/MTJqVx0SC4epBq+N2yhuwtNwLbJYNZyhL2pO1ZVKn5HFXav5T41Ryzy9K9V7ZId2JB2oy/W4aCd9/2w==", + "version": "8.57.1", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.1.tgz", + "integrity": "sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q==", "dev": true, + "license": "MIT", "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, "node_modules/@humanwhocodes/config-array": { - "version": "0.11.11", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.11.tgz", - "integrity": "sha512-N2brEuAadi0CcdeMXUkhbZB84eskAc8MEX1By6qEchoVywSgXPIjou4rYsl0V3Hj0ZnuGycGCjdNgockbzeWNA==", + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.13.0.tgz", + "integrity": "sha512-DZLEEqFWQFiyK6h5YIeynKx7JlvCYWL0cImfSRXZ9l4Sg2efkFGTuFf6vzXjK1cq6IYkU+Eg/JizXw+TD2vRNw==", + "deprecated": "Use @eslint/config-array instead", "dev": true, + "license": "Apache-2.0", "dependencies": { - "@humanwhocodes/object-schema": "^1.2.1", - "debug": "^4.1.1", + "@humanwhocodes/object-schema": "^2.0.3", + "debug": "^4.3.1", "minimatch": "^3.0.5" }, "engines": { "node": ">=10.10.0" } }, + "node_modules/@humanwhocodes/config-array/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/@humanwhocodes/config-array/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/@humanwhocodes/module-importer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", "dev": true, + "license": "Apache-2.0", "engines": { "node": ">=12.22" }, @@ -125,16 +174,19 @@ } }, "node_modules/@humanwhocodes/object-schema": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz", - "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", - "dev": true + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz", + "integrity": "sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==", + "deprecated": "Use @eslint/object-schema instead", + "dev": true, + "license": "BSD-3-Clause" }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", "dev": true, + "license": "MIT", "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" @@ -148,6 +200,7 @@ "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", "dev": true, + "license": "MIT", "engines": { "node": ">= 8" } @@ -157,6 +210,7 @@ "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", "dev": true, + "license": "MIT", "dependencies": { "@nodelib/fs.scandir": "2.1.5", "fastq": "^1.6.0" @@ -166,34 +220,41 @@ } }, "node_modules/@types/json-schema": { - "version": "7.0.12", - "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.12.tgz", - "integrity": "sha512-Hr5Jfhc9eYOQNPYO5WLDq/n4jqijdHNlDXjuAQkkt+mWdQR+XJToOHrsD4cPaMXpn6KO7y2+wM8AZEs8VpBLVA==", - "dev": true + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true, + "license": "MIT" }, "node_modules/@types/node": { - "version": "20.5.7", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.5.7.tgz", - "integrity": "sha512-dP7f3LdZIysZnmvP3ANJYTSwg+wLLl8p7RqniVlV7j+oXSXAbt9h0WIBFmJy5inWZoX9wZN6eXx+YXd9Rh3RBA==", - "dev": true + "version": "20.17.6", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.17.6.tgz", + "integrity": "sha512-VEI7OdvK2wP7XHnsuXbAJnEpEkF6NjSN45QJlL4VGqZSXsnicpesdTWsg9RISeSdYd3yeRj/y3k5KGjUXYnFwQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.19.2" + } }, "node_modules/@types/semver": { - "version": "7.5.2", - "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.2.tgz", - "integrity": "sha512-7aqorHYgdNO4DM36stTiGO3DvKoex9TQRwsJU6vMaFGyqpBA1MNZkz+PG3gaNUPpTAOYhT1WR7M1JyA3fbS9Cw==", - "dev": true + "version": "7.5.8", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.8.tgz", + "integrity": "sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ==", + "dev": true, + "license": "MIT" }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "6.7.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.7.0.tgz", - "integrity": "sha512-gUqtknHm0TDs1LhY12K2NA3Rmlmp88jK9Tx8vGZMfHeNMLE3GH2e9TRub+y+SOjuYgtOmok+wt1AyDPZqxbNag==", + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.21.0.tgz", + "integrity": "sha512-oy9+hTPCUFpngkEZUSzbf9MxI65wbKFoQYsgPdILTfbUldp5ovUuphZVe4i30emU9M/kP+T64Di0mxl7dSw3MA==", "dev": true, + "license": "MIT", "dependencies": { "@eslint-community/regexpp": "^4.5.1", - "@typescript-eslint/scope-manager": "6.7.0", - "@typescript-eslint/type-utils": "6.7.0", - "@typescript-eslint/utils": "6.7.0", - "@typescript-eslint/visitor-keys": "6.7.0", + "@typescript-eslint/scope-manager": "6.21.0", + "@typescript-eslint/type-utils": "6.21.0", + "@typescript-eslint/utils": "6.21.0", + "@typescript-eslint/visitor-keys": "6.21.0", "debug": "^4.3.4", "graphemer": "^1.4.0", "ignore": "^5.2.4", @@ -219,15 +280,16 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "6.7.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-6.7.0.tgz", - "integrity": "sha512-jZKYwqNpNm5kzPVP5z1JXAuxjtl2uG+5NpaMocFPTNC2EdYIgbXIPImObOkhbONxtFTTdoZstLZefbaK+wXZng==", + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-6.21.0.tgz", + "integrity": "sha512-tbsV1jPne5CkFQCgPBcDOt30ItF7aJoZL997JSF7MhGQqOeT3svWRYxiqlfA5RUdlHN6Fi+EI9bxqbdyAUZjYQ==", "dev": true, + "license": "BSD-2-Clause", "dependencies": { - "@typescript-eslint/scope-manager": "6.7.0", - "@typescript-eslint/types": "6.7.0", - "@typescript-eslint/typescript-estree": "6.7.0", - "@typescript-eslint/visitor-keys": "6.7.0", + "@typescript-eslint/scope-manager": "6.21.0", + "@typescript-eslint/types": "6.21.0", + "@typescript-eslint/typescript-estree": "6.21.0", + "@typescript-eslint/visitor-keys": "6.21.0", "debug": "^4.3.4" }, "engines": { @@ -247,13 +309,14 @@ } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "6.7.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-6.7.0.tgz", - "integrity": "sha512-lAT1Uau20lQyjoLUQ5FUMSX/dS07qux9rYd5FGzKz/Kf8W8ccuvMyldb8hadHdK/qOI7aikvQWqulnEq2nCEYA==", + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-6.21.0.tgz", + "integrity": "sha512-OwLUIWZJry80O99zvqXVEioyniJMa+d2GrqpUTqi5/v5D5rOrppJVBPa0yKCblcigC0/aYAzxxqQ1B+DS2RYsg==", "dev": true, + "license": "MIT", "dependencies": { - "@typescript-eslint/types": "6.7.0", - "@typescript-eslint/visitor-keys": "6.7.0" + "@typescript-eslint/types": "6.21.0", + "@typescript-eslint/visitor-keys": "6.21.0" }, "engines": { "node": "^16.0.0 || >=18.0.0" @@ -264,13 +327,14 @@ } }, "node_modules/@typescript-eslint/type-utils": { - "version": "6.7.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-6.7.0.tgz", - "integrity": "sha512-f/QabJgDAlpSz3qduCyQT0Fw7hHpmhOzY/Rv6zO3yO+HVIdPfIWhrQoAyG+uZVtWAIS85zAyzgAFfyEr+MgBpg==", + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-6.21.0.tgz", + "integrity": "sha512-rZQI7wHfao8qMX3Rd3xqeYSMCL3SoiSQLBATSiVKARdFGCYSRvmViieZjqc58jKgs8Y8i9YvVVhRbHSTA4VBag==", "dev": true, + "license": "MIT", "dependencies": { - "@typescript-eslint/typescript-estree": "6.7.0", - "@typescript-eslint/utils": "6.7.0", + "@typescript-eslint/typescript-estree": "6.21.0", + "@typescript-eslint/utils": "6.21.0", "debug": "^4.3.4", "ts-api-utils": "^1.0.1" }, @@ -291,10 +355,11 @@ } }, "node_modules/@typescript-eslint/types": { - "version": "6.7.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-6.7.0.tgz", - "integrity": "sha512-ihPfvOp7pOcN/ysoj0RpBPOx3HQTJTrIN8UZK+WFd3/iDeFHHqeyYxa4hQk4rMhsz9H9mXpR61IzwlBVGXtl9Q==", + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-6.21.0.tgz", + "integrity": "sha512-1kFmZ1rOm5epu9NZEZm1kckCDGj5UJEf7P1kliH4LKu/RkwpsfqqGmY2OOcUs18lSlQBKLDYBOGxRVtrMN5lpg==", "dev": true, + "license": "MIT", "engines": { "node": "^16.0.0 || >=18.0.0" }, @@ -304,16 +369,18 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "6.7.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-6.7.0.tgz", - "integrity": "sha512-dPvkXj3n6e9yd/0LfojNU8VMUGHWiLuBZvbM6V6QYD+2qxqInE7J+J/ieY2iGwR9ivf/R/haWGkIj04WVUeiSQ==", + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-6.21.0.tgz", + "integrity": "sha512-6npJTkZcO+y2/kr+z0hc4HwNfrrP4kNYh57ek7yCNlrBjWQ1Y0OS7jiZTkgumrvkX5HkEKXFZkkdFNkaW2wmUQ==", "dev": true, + "license": "BSD-2-Clause", "dependencies": { - "@typescript-eslint/types": "6.7.0", - "@typescript-eslint/visitor-keys": "6.7.0", + "@typescript-eslint/types": "6.21.0", + "@typescript-eslint/visitor-keys": "6.21.0", "debug": "^4.3.4", "globby": "^11.1.0", "is-glob": "^4.0.3", + "minimatch": "9.0.3", "semver": "^7.5.4", "ts-api-utils": "^1.0.1" }, @@ -331,17 +398,18 @@ } }, "node_modules/@typescript-eslint/utils": { - "version": "6.7.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-6.7.0.tgz", - "integrity": "sha512-MfCq3cM0vh2slSikQYqK2Gq52gvOhe57vD2RM3V4gQRZYX4rDPnKLu5p6cm89+LJiGlwEXU8hkYxhqqEC/V3qA==", + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-6.21.0.tgz", + "integrity": "sha512-NfWVaC8HP9T8cbKQxHcsJBY5YE1O33+jpMwN45qzWWaPDZgLIbo12toGMWnmhvCpd3sIxkpDw3Wv1B3dYrbDQQ==", "dev": true, + "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.4.0", "@types/json-schema": "^7.0.12", "@types/semver": "^7.5.0", - "@typescript-eslint/scope-manager": "6.7.0", - "@typescript-eslint/types": "6.7.0", - "@typescript-eslint/typescript-estree": "6.7.0", + "@typescript-eslint/scope-manager": "6.21.0", + "@typescript-eslint/types": "6.21.0", + "@typescript-eslint/typescript-estree": "6.21.0", "semver": "^7.5.4" }, "engines": { @@ -356,12 +424,13 @@ } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "6.7.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-6.7.0.tgz", - "integrity": "sha512-/C1RVgKFDmGMcVGeD8HjKv2bd72oI1KxQDeY8uc66gw9R0OK0eMq48cA+jv9/2Ag6cdrsUGySm1yzYmfz0hxwQ==", + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-6.21.0.tgz", + "integrity": "sha512-JJtkDduxLi9bivAB+cYOVMtbkqdPOhZ+ZI5LC47MIRrDV4Yn2o+ZnW10Nkmr28xRpSpdJ6Sm42Hjf2+REYXm0A==", "dev": true, + "license": "MIT", "dependencies": { - "@typescript-eslint/types": "6.7.0", + "@typescript-eslint/types": "6.21.0", "eslint-visitor-keys": "^3.4.1" }, "engines": { @@ -372,11 +441,19 @@ "url": "https://opencollective.com/typescript-eslint" } }, + "node_modules/@ungap/structured-clone": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", + "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==", + "dev": true, + "license": "ISC" + }, "node_modules/acorn": { - "version": "8.10.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.10.0.tgz", - "integrity": "sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw==", + "version": "8.14.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.0.tgz", + "integrity": "sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==", "dev": true, + "license": "MIT", "bin": { "acorn": "bin/acorn" }, @@ -389,14 +466,16 @@ "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", "dev": true, + "license": "MIT", "peerDependencies": { "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, "node_modules/agent-base": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.0.tgz", - "integrity": "sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", + "license": "MIT", "dependencies": { "debug": "^4.3.4" }, @@ -409,6 +488,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", "dev": true, + "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -425,6 +505,7 @@ "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz", "integrity": "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==", "dev": true, + "license": "MIT", "engines": { "node": ">=6" } @@ -434,6 +515,7 @@ "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" } @@ -443,6 +525,7 @@ "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", "dev": true, + "license": "MIT", "dependencies": { "color-convert": "^2.0.1" }, @@ -458,6 +541,7 @@ "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", "dev": true, + "license": "ISC", "dependencies": { "normalize-path": "^3.0.0", "picomatch": "^2.0.4" @@ -470,33 +554,84 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true + "dev": true, + "license": "Python-2.0" }, "node_modules/array-union": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/b4a": { - "version": "1.6.6", - "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.6.tgz", - "integrity": "sha512-5Tk1HLk6b6ctmjIkAcU/Ujv/1WqiDl0F0JdRCR80VsOcUlHcu7pWeWRlOqQLHfDEsVx9YH/aif5AG4ehoCtTmg==" + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.7.tgz", + "integrity": "sha512-OnAYlL5b7LEkALw87fUVafQw5rVR9RjwGd4KUwNQ6DrrNmaVaUCgLipfVlzrPQ4tWOR9P0IXGNOx50jYCCdSJg==", + "license": "Apache-2.0" }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true + "dev": true, + "license": "MIT" + }, + "node_modules/bare-events": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.5.0.tgz", + "integrity": "sha512-/E8dDe9dsbLyh2qrZ64PEPadOQ0F4gbl1sUJOrmph7xOiIxfY8vwab/4bFLh4Y88/Hk/ujKcrQKc+ps0mv873A==", + "license": "Apache-2.0", + "optional": true + }, + "node_modules/bare-fs": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-2.3.5.tgz", + "integrity": "sha512-SlE9eTxifPDJrT6YgemQ1WGFleevzwY+XAP1Xqgl56HtcrisC2CHCZ2tq6dBpcH2TnNxwUEUGhweo+lrQtYuiw==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "bare-events": "^2.0.0", + "bare-path": "^2.0.0", + "bare-stream": "^2.0.0" + } + }, + "node_modules/bare-os": { + "version": "2.4.4", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-2.4.4.tgz", + "integrity": "sha512-z3UiI2yi1mK0sXeRdc4O1Kk8aOa/e+FNWZcTiPB/dfTWyLypuE99LibgRaQki914Jq//yAWylcAt+mknKdixRQ==", + "license": "Apache-2.0", + "optional": true + }, + "node_modules/bare-path": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-2.1.3.tgz", + "integrity": "sha512-lh/eITfU8hrj9Ru5quUp0Io1kJWIk1bTjzo7JH1P5dWmQ2EL4hFUlfI8FonAhSlgIfhn63p84CDY/x+PisgcXA==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "bare-os": "^2.1.0" + } + }, + "node_modules/bare-stream": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.3.2.tgz", + "integrity": "sha512-EFZHSIBkDgSHIwj2l2QZfP4U5OcD4xFAOwhSb/vlr9PIqyGJGvB/nfClJbcnh3EY4jtPE4zsb5ztae96bVF79A==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "streamx": "^2.20.0" + } }, "node_modules/binary-extensions": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" }, @@ -505,13 +640,13 @@ } }, "node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", "dev": true, + "license": "MIT", "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" + "balanced-match": "^1.0.0" } }, "node_modules/braces": { @@ -519,6 +654,7 @@ "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dev": true, + "license": "MIT", "dependencies": { "fill-range": "^7.1.1" }, @@ -530,12 +666,14 @@ "version": "1.3.1", "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", - "dev": true + "dev": true, + "license": "ISC" }, "node_modules/browserify-zlib": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/browserify-zlib/-/browserify-zlib-0.1.4.tgz", "integrity": "sha512-19OEpq7vWgsH6WkvkBJQDFvJS1uPcbFOQ4v9CU839dO+ZZXUZO6XpE6hNCqvlIIj+4fZvRiJ6DsAQ382GwiyTQ==", + "license": "MIT", "dependencies": { "pako": "~0.2.0" } @@ -543,13 +681,15 @@ "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", - "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "license": "MIT" }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", "dev": true, + "license": "MIT", "engines": { "node": ">=6" } @@ -559,6 +699,7 @@ "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", "dev": true, + "license": "MIT", "engines": { "node": ">=10" }, @@ -571,6 +712,7 @@ "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", "dev": true, + "license": "MIT", "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" @@ -587,6 +729,7 @@ "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", "dev": true, + "license": "MIT", "dependencies": { "anymatch": "~3.1.2", "braces": "~3.0.2", @@ -611,6 +754,7 @@ "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", "dev": true, + "license": "ISC", "dependencies": { "is-glob": "^4.0.1" }, @@ -623,6 +767,7 @@ "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", "dev": true, + "license": "ISC", "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.0", @@ -634,6 +779,7 @@ "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", "dev": true, + "license": "MIT", "dependencies": { "color-name": "~1.1.4" }, @@ -645,24 +791,28 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/core-util-is": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", - "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "license": "MIT" }, "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dev": true, + "license": "MIT", "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -673,11 +823,12 @@ } }, "node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "license": "MIT", "dependencies": { - "ms": "2.1.2" + "ms": "^2.1.3" }, "engines": { "node": ">=6.0" @@ -693,6 +844,7 @@ "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-4.0.0.tgz", "integrity": "sha512-9iE1PgSik9HeIIw2JO94IidnE3eBoQrFJ3w7sFuzSX4DpmZ3v5sZpUiV5Swcf6mQEF+Y0ru8Neo+p+nyh2J+hQ==", "dev": true, + "license": "MIT", "engines": { "node": ">=10" }, @@ -704,13 +856,15 @@ "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/diff": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/diff/-/diff-5.2.0.tgz", "integrity": "sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A==", "dev": true, + "license": "BSD-3-Clause", "engines": { "node": ">=0.3.1" } @@ -720,6 +874,7 @@ "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", "dev": true, + "license": "MIT", "dependencies": { "path-type": "^4.0.0" }, @@ -732,6 +887,7 @@ "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", "dev": true, + "license": "Apache-2.0", "dependencies": { "esutils": "^2.0.2" }, @@ -743,6 +899,7 @@ "version": "3.7.1", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==", + "license": "MIT", "dependencies": { "end-of-stream": "^1.0.0", "inherits": "^2.0.1", @@ -754,21 +911,24 @@ "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/end-of-stream": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "license": "MIT", "dependencies": { "once": "^1.4.0" } }, "node_modules/escalade": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz", - "integrity": "sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", "dev": true, + "license": "MIT", "engines": { "node": ">=6" } @@ -778,6 +938,7 @@ "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", "dev": true, + "license": "MIT", "engines": { "node": ">=10" }, @@ -786,18 +947,21 @@ } }, "node_modules/eslint": { - "version": "8.49.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.49.0.tgz", - "integrity": "sha512-jw03ENfm6VJI0jA9U+8H5zfl5b+FvuU3YYvZRdZHOlU2ggJkxrlkJH4HcDrZpj6YwD8kuYqvQM8LyesoazrSOQ==", + "version": "8.57.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.1.tgz", + "integrity": "sha512-ypowyDxpVSYpkXr9WPv2PAZCtNip1Mv5KTW0SCurXv/9iOpcrH9PaqUElksqEB6pChqHGDRCFTyrZlGhnLNGiA==", + "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", "dev": true, + "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", - "@eslint/eslintrc": "^2.1.2", - "@eslint/js": "8.49.0", - "@humanwhocodes/config-array": "^0.11.11", + "@eslint/eslintrc": "^2.1.4", + "@eslint/js": "8.57.1", + "@humanwhocodes/config-array": "^0.13.0", "@humanwhocodes/module-importer": "^1.0.1", "@nodelib/fs.walk": "^1.2.8", + "@ungap/structured-clone": "^1.2.0", "ajv": "^6.12.4", "chalk": "^4.0.0", "cross-spawn": "^7.0.2", @@ -844,6 +1008,7 @@ "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==", "dev": true, + "license": "BSD-2-Clause", "dependencies": { "esrecurse": "^4.3.0", "estraverse": "^5.2.0" @@ -860,6 +1025,7 @@ "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", "dev": true, + "license": "Apache-2.0", "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, @@ -867,11 +1033,36 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/eslint/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/eslint/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/espree": { "version": "9.6.1", "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", "integrity": "sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==", "dev": true, + "license": "BSD-2-Clause", "dependencies": { "acorn": "^8.9.0", "acorn-jsx": "^5.3.2", @@ -885,10 +1076,11 @@ } }, "node_modules/esquery": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz", - "integrity": "sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.6.0.tgz", + "integrity": "sha512-ca9pw9fomFcKPvFLXhBKUK90ZvGibiGOvRJNbjljY7s7uq/5YO4BOzcYtJqExdx99rF6aAcnRxHmcUHcz6sQsg==", "dev": true, + "license": "BSD-3-Clause", "dependencies": { "estraverse": "^5.1.0" }, @@ -901,6 +1093,7 @@ "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", "dev": true, + "license": "BSD-2-Clause", "dependencies": { "estraverse": "^5.2.0" }, @@ -913,6 +1106,7 @@ "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", "dev": true, + "license": "BSD-2-Clause", "engines": { "node": ">=4.0" } @@ -922,6 +1116,7 @@ "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", "dev": true, + "license": "BSD-2-Clause", "engines": { "node": ">=0.10.0" } @@ -930,18 +1125,21 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/fast-fifo": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", - "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==" + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "license": "MIT" }, "node_modules/fast-glob": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.1.tgz", - "integrity": "sha512-kNFPyjhh5cKjrUltxs+wFx+ZkbRaxxmZ+X0ZU31SOsxCEtP9VPgtq2teZw1DebupL5GmDaNQ6yKMMVcM41iqDg==", + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", + "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==", "dev": true, + "license": "MIT", "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", @@ -958,6 +1156,7 @@ "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", "dev": true, + "license": "ISC", "dependencies": { "is-glob": "^4.0.1" }, @@ -969,19 +1168,22 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/fast-levenshtein": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/fastq": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", - "integrity": "sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==", + "version": "1.17.1", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz", + "integrity": "sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==", "dev": true, + "license": "ISC", "dependencies": { "reusify": "^1.0.4" } @@ -991,6 +1193,7 @@ "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", "dev": true, + "license": "MIT", "dependencies": { "flat-cache": "^3.0.4" }, @@ -1003,6 +1206,7 @@ "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dev": true, + "license": "MIT", "dependencies": { "to-regex-range": "^5.0.1" }, @@ -1015,6 +1219,7 @@ "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", "dev": true, + "license": "MIT", "dependencies": { "locate-path": "^6.0.0", "path-exists": "^4.0.0" @@ -1031,35 +1236,39 @@ "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz", "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==", "dev": true, + "license": "BSD-3-Clause", "bin": { "flat": "cli.js" } }, "node_modules/flat-cache": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.1.0.tgz", - "integrity": "sha512-OHx4Qwrrt0E4jEIcI5/Xb+f+QmJYNj2rrK8wiIdQOIrB9WrrJL8cjZvXdXuBTkkEwEqLycb5BeZDV1o2i9bTew==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.2.0.tgz", + "integrity": "sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==", "dev": true, + "license": "MIT", "dependencies": { - "flatted": "^3.2.7", + "flatted": "^3.2.9", "keyv": "^4.5.3", "rimraf": "^3.0.2" }, "engines": { - "node": ">=12.0.0" + "node": "^10.12.0 || >=12.0.0" } }, "node_modules/flatted": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.7.tgz", - "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==", - "dev": true + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.1.tgz", + "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==", + "dev": true, + "license": "ISC" }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true + "dev": true, + "license": "ISC" }, "node_modules/fsevents": { "version": "2.3.3", @@ -1067,6 +1276,7 @@ "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", "dev": true, "hasInstallScript": true, + "license": "MIT", "optional": true, "os": [ "darwin" @@ -1080,25 +1290,27 @@ "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", "dev": true, + "license": "ISC", "engines": { "node": "6.* || 8.* || >= 10.*" } }, "node_modules/glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", + "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", + "deprecated": "Glob versions prior to v9 are no longer supported", "dev": true, + "license": "ISC", "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" + "minimatch": "^5.0.1", + "once": "^1.3.0" }, "engines": { - "node": "*" + "node": ">=12" }, "funding": { "url": "https://github.com/sponsors/isaacs" @@ -1109,6 +1321,7 @@ "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", "dev": true, + "license": "ISC", "dependencies": { "is-glob": "^4.0.3" }, @@ -1116,11 +1329,25 @@ "node": ">=10.13.0" } }, + "node_modules/glob/node_modules/minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/globals": { - "version": "13.21.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.21.0.tgz", - "integrity": "sha512-ybyme3s4yy/t/3s35bewwXKOf7cvzfreG2lH0lZl0JB7I4GxRP2ghxOK/Nb9EkRXdbBXZLfq/p/0W2JUONB/Gg==", + "version": "13.24.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", + "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", "dev": true, + "license": "MIT", "dependencies": { "type-fest": "^0.20.2" }, @@ -1136,6 +1363,7 @@ "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", "dev": true, + "license": "MIT", "dependencies": { "array-union": "^2.1.0", "dir-glob": "^3.0.1", @@ -1155,12 +1383,14 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/gunzip-maybe": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/gunzip-maybe/-/gunzip-maybe-1.4.2.tgz", "integrity": "sha512-4haO1M4mLO91PW57BMsDFf75UmwoRX0GkdD+Faw+Lr+r/OZrOCS0pIBwOL1xCKQqnQzbNFGgK2V2CpBUPeFNTw==", + "license": "MIT", "dependencies": { "browserify-zlib": "^0.1.4", "is-deflate": "^1.0.0", @@ -1178,6 +1408,7 @@ "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" } @@ -1187,14 +1418,16 @@ "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", "dev": true, + "license": "MIT", "bin": { "he": "bin/he" } }, "node_modules/https-proxy-agent": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.2.tgz", - "integrity": "sha512-NmLNjm6ucYwtcUmL7JQC1ZQ57LmHP4lT15FQ8D61nak1rO6DH+fz5qNK2Ap5UN4ZapYICE3/0KodcLYSPsPbaA==", + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz", + "integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==", + "license": "MIT", "dependencies": { "agent-base": "^7.0.2", "debug": "4" @@ -1204,10 +1437,11 @@ } }, "node_modules/ignore": { - "version": "5.2.4", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz", - "integrity": "sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ==", + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", "dev": true, + "license": "MIT", "engines": { "node": ">= 4" } @@ -1217,6 +1451,7 @@ "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", "dev": true, + "license": "MIT", "dependencies": { "parent-module": "^1.0.0", "resolve-from": "^4.0.0" @@ -1233,6 +1468,7 @@ "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", "dev": true, + "license": "MIT", "engines": { "node": ">=0.8.19" } @@ -1241,7 +1477,9 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", "dev": true, + "license": "ISC", "dependencies": { "once": "^1.3.0", "wrappy": "1" @@ -1250,13 +1488,15 @@ "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" }, "node_modules/is-binary-path": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", "dev": true, + "license": "MIT", "dependencies": { "binary-extensions": "^2.0.0" }, @@ -1267,13 +1507,15 @@ "node_modules/is-deflate": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-deflate/-/is-deflate-1.0.0.tgz", - "integrity": "sha512-YDoFpuZWu1VRXlsnlYMzKyVRITXj7Ej/V9gXQ2/pAe7X1J7M/RNOqaIYi6qUn+B7nGyB9pDXrv02dsB58d2ZAQ==" + "integrity": "sha512-YDoFpuZWu1VRXlsnlYMzKyVRITXj7Ej/V9gXQ2/pAe7X1J7M/RNOqaIYi6qUn+B7nGyB9pDXrv02dsB58d2ZAQ==", + "license": "MIT" }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", "dev": true, + "license": "MIT", "engines": { "node": ">=0.10.0" } @@ -1283,6 +1525,7 @@ "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" } @@ -1292,6 +1535,7 @@ "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", "dev": true, + "license": "MIT", "dependencies": { "is-extglob": "^2.1.1" }, @@ -1303,6 +1547,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-gzip/-/is-gzip-1.0.0.tgz", "integrity": "sha512-rcfALRIb1YewtnksfRIHGcIY93QnK8BIQ/2c9yDYcG/Y6+vRoJuTWBmmSEbyLLYtXm7q35pHOHbZFQBaLrhlWQ==", + "license": "MIT", "engines": { "node": ">=0.10.0" } @@ -1312,6 +1557,7 @@ "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", "dev": true, + "license": "MIT", "engines": { "node": ">=0.12.0" } @@ -1321,6 +1567,7 @@ "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" } @@ -1330,6 +1577,7 @@ "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-2.1.0.tgz", "integrity": "sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" } @@ -1339,6 +1587,7 @@ "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==", "dev": true, + "license": "MIT", "engines": { "node": ">=10" }, @@ -1349,19 +1598,22 @@ "node_modules/isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "dev": true + "dev": true, + "license": "ISC" }, "node_modules/js-yaml": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", "dev": true, + "license": "MIT", "dependencies": { "argparse": "^2.0.1" }, @@ -1373,25 +1625,29 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/keyv": { - "version": "4.5.3", - "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.3.tgz", - "integrity": "sha512-QCiSav9WaX1PgETJ+SpNnx2PRRapJ/oRSXM4VO5OGYGSjrxbKPVFVhB3l2OCbLCk329N8qyAtsJjSjvVBWzEug==", + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", "dev": true, + "license": "MIT", "dependencies": { "json-buffer": "3.0.1" } @@ -1401,6 +1657,7 @@ "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", "dev": true, + "license": "MIT", "dependencies": { "prelude-ls": "^1.2.1", "type-check": "~0.4.0" @@ -1414,6 +1671,7 @@ "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", "dev": true, + "license": "MIT", "dependencies": { "p-locate": "^5.0.0" }, @@ -1428,13 +1686,15 @@ "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/log-symbols": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", "dev": true, + "license": "MIT", "dependencies": { "chalk": "^4.1.0", "is-unicode-supported": "^0.1.0" @@ -1451,6 +1711,7 @@ "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", "dev": true, + "license": "MIT", "engines": { "node": ">= 8" } @@ -1460,6 +1721,7 @@ "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", "dev": true, + "license": "MIT", "dependencies": { "braces": "^3.0.3", "picomatch": "^2.3.1" @@ -1469,27 +1731,27 @@ } }, "node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz", + "integrity": "sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==", "dev": true, + "license": "ISC", "dependencies": { - "brace-expansion": "^1.1.7" + "brace-expansion": "^2.0.1" }, "engines": { - "node": "*" + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/mkdirp-classic": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", - "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" - }, "node_modules/mocha": { - "version": "10.6.0", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-10.6.0.tgz", - "integrity": "sha512-hxjt4+EEB0SA0ZDygSS015t65lJw/I2yRCS3Ae+SJ5FrbzrXgfYwJr96f0OvIXdj7h4lv/vLCrH3rkiuizFSvw==", + "version": "10.8.2", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-10.8.2.tgz", + "integrity": "sha512-VZlYo/WE8t1tstuRmqgeyBgCbJc/lEdopaa+axcKzTBJ+UIdlAB9XnmvTCAH4pwR4ElNInaedhEBmZD8iCSVEg==", "dev": true, + "license": "MIT", "dependencies": { "ansi-colors": "^4.1.3", "browser-stdout": "^1.3.1", @@ -1520,40 +1782,12 @@ "node": ">= 14.0.0" } }, - "node_modules/mocha/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dev": true, - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/mocha/node_modules/glob": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", - "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", - "deprecated": "Glob versions prior to v9 are no longer supported", - "dev": true, - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^5.0.1", - "once": "^1.3.0" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/mocha/node_modules/minimatch": { "version": "5.1.6", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", "dev": true, + "license": "ISC", "dependencies": { "brace-expansion": "^2.0.1" }, @@ -1561,17 +1795,12 @@ "node": ">=10" } }, - "node_modules/mocha/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true - }, "node_modules/mocha/node_modules/supports-color": { "version": "8.1.1", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", "dev": true, + "license": "MIT", "dependencies": { "has-flag": "^4.0.0" }, @@ -1583,21 +1812,24 @@ } }, "node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", "dev": true, + "license": "MIT", "engines": { "node": ">=0.10.0" } @@ -1606,22 +1838,24 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", "dependencies": { "wrappy": "1" } }, "node_modules/optionator": { - "version": "0.9.3", - "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz", - "integrity": "sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg==", + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", + "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", "dev": true, + "license": "MIT", "dependencies": { - "@aashutoshrathi/word-wrap": "^1.2.3", "deep-is": "^0.1.3", "fast-levenshtein": "^2.0.6", "levn": "^0.4.1", "prelude-ls": "^1.2.1", - "type-check": "^0.4.0" + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" }, "engines": { "node": ">= 0.8.0" @@ -1632,6 +1866,7 @@ "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", "dev": true, + "license": "MIT", "dependencies": { "yocto-queue": "^0.1.0" }, @@ -1647,6 +1882,7 @@ "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", "dev": true, + "license": "MIT", "dependencies": { "p-limit": "^3.0.2" }, @@ -1660,13 +1896,15 @@ "node_modules/pako": { "version": "0.2.9", "resolved": "https://registry.npmjs.org/pako/-/pako-0.2.9.tgz", - "integrity": "sha512-NUcwaKxUxWrZLpDG+z/xZaCgQITkA/Dv4V/T6bw7VON6l1Xz/VnrBqrYjZQ12TamKHzITTfOEIYUj48y2KXImA==" + "integrity": "sha512-NUcwaKxUxWrZLpDG+z/xZaCgQITkA/Dv4V/T6bw7VON6l1Xz/VnrBqrYjZQ12TamKHzITTfOEIYUj48y2KXImA==", + "license": "MIT" }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", "dev": true, + "license": "MIT", "dependencies": { "callsites": "^3.0.0" }, @@ -1679,6 +1917,7 @@ "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" } @@ -1688,6 +1927,7 @@ "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", "dev": true, + "license": "MIT", "engines": { "node": ">=0.10.0" } @@ -1697,6 +1937,7 @@ "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" } @@ -1706,6 +1947,7 @@ "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" } @@ -1714,6 +1956,7 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/peek-stream/-/peek-stream-1.1.3.tgz", "integrity": "sha512-FhJ+YbOSBb9/rIl2ZeE/QHEsWn7PqNYt8ARAY3kIgNGOk13g9FGyIY6JIl/xB/3TFRVoTv5as0l11weORrTekA==", + "license": "MIT", "dependencies": { "buffer-from": "^1.0.0", "duplexify": "^3.5.0", @@ -1725,6 +1968,7 @@ "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", "dev": true, + "license": "MIT", "engines": { "node": ">=8.6" }, @@ -1737,6 +1981,7 @@ "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.8.0" } @@ -1744,12 +1989,14 @@ "node_modules/process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "license": "MIT" }, "node_modules/pump": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", - "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", + "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==", + "license": "MIT", "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -1759,26 +2006,19 @@ "version": "1.5.1", "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-1.5.1.tgz", "integrity": "sha512-oClZI37HvuUJJxSKKrC17bZ9Cu0ZYhEAGPsPUy9KlMUmv9dKX2o77RUmq7f3XjIxbwyGwYzbzQ1L2Ks8sIradQ==", + "license": "MIT", "dependencies": { "duplexify": "^3.6.0", "inherits": "^2.0.3", "pump": "^2.0.0" } }, - "node_modules/pumpify/node_modules/pump": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", - "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==", - "dependencies": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - }, "node_modules/punycode": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", - "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", "dev": true, + "license": "MIT", "engines": { "node": ">=6" } @@ -1801,18 +2041,21 @@ "type": "consulting", "url": "https://feross.org/support" } - ] + ], + "license": "MIT" }, "node_modules/queue-tick": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz", - "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==" + "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==", + "license": "MIT" }, "node_modules/random-bigint": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/random-bigint/-/random-bigint-0.0.1.tgz", "integrity": "sha512-X+NTsf5Hzl/tRNLiNTD3N1LRU0eKdIE0+plNlV1CmXLTlnAxj6HipcTnOhWvFRoSytCz6l1f4KYFf/iH8NNSLw==", "dev": true, + "license": "ISC", "engines": { "node": ">=10.0.0" } @@ -1822,6 +2065,7 @@ "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", "dev": true, + "license": "MIT", "dependencies": { "safe-buffer": "^5.1.0" } @@ -1830,6 +2074,7 @@ "version": "2.3.8", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -1845,6 +2090,7 @@ "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", "dev": true, + "license": "MIT", "dependencies": { "picomatch": "^2.2.1" }, @@ -1857,6 +2103,7 @@ "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", "dev": true, + "license": "MIT", "engines": { "node": ">=0.10.0" } @@ -1866,6 +2113,7 @@ "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", "dev": true, + "license": "MIT", "engines": { "node": ">=4" } @@ -1875,6 +2123,7 @@ "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", "dev": true, + "license": "MIT", "engines": { "iojs": ">=1.0.0", "node": ">=0.10.0" @@ -1884,7 +2133,9 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", "dev": true, + "license": "ISC", "dependencies": { "glob": "^7.1.3" }, @@ -1895,6 +2146,52 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/rimraf/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/rimraf/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dev": true, + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/rimraf/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", @@ -1914,6 +2211,7 @@ "url": "https://feross.org/support" } ], + "license": "MIT", "dependencies": { "queue-microtask": "^1.2.2" } @@ -1921,16 +2219,15 @@ "node_modules/safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" }, "node_modules/semver": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", - "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", "dev": true, - "dependencies": { - "lru-cache": "^6.0.0" - }, + "license": "ISC", "bin": { "semver": "bin/semver.js" }, @@ -1938,23 +2235,12 @@ "node": ">=10" } }, - "node_modules/semver/node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dev": true, - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/serialize-javascript": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.2.tgz", "integrity": "sha512-Saa1xPByTTq2gdeFZYLLo+RFE35NHZkAbqZeWNd3BpzppeVisAqpDjcp8dyf6uIvEqJRd46jemmyA4iFIeVk8g==", "dev": true, + "license": "BSD-3-Clause", "dependencies": { "randombytes": "^2.1.0" } @@ -1964,6 +2250,7 @@ "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", "dev": true, + "license": "MIT", "dependencies": { "shebang-regex": "^3.0.0" }, @@ -1976,6 +2263,7 @@ "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" } @@ -1985,6 +2273,7 @@ "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" } @@ -1992,21 +2281,28 @@ "node_modules/stream-shift": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", - "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==" + "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==", + "license": "MIT" }, "node_modules/streamx": { - "version": "2.15.7", - "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.15.7.tgz", - "integrity": "sha512-NPEKS5+yjyo597eafGbKW5ujh7Sm6lDLHZQd/lRSz6S0VarpADBJItqfB4PnwpS+472oob1GX5cCY9vzfJpHUA==", + "version": "2.20.2", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.20.2.tgz", + "integrity": "sha512-aDGDLU+j9tJcUdPGOaHmVF1u/hhI+CsGkT02V3OKlHDV7IukOI+nTWAGkiZEKCO35rWN1wIr4tS7YFr1f4qSvA==", + "license": "MIT", "dependencies": { - "fast-fifo": "^1.1.0", - "queue-tick": "^1.0.1" + "fast-fifo": "^1.3.2", + "queue-tick": "^1.0.1", + "text-decoder": "^1.1.0" + }, + "optionalDependencies": { + "bare-events": "^2.2.0" } }, "node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", "dependencies": { "safe-buffer": "~5.1.0" } @@ -2016,6 +2312,7 @@ "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "dev": true, + "license": "MIT", "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -2030,6 +2327,7 @@ "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "dev": true, + "license": "MIT", "dependencies": { "ansi-regex": "^5.0.1" }, @@ -2042,6 +2340,7 @@ "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" }, @@ -2054,6 +2353,7 @@ "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", "dev": true, + "license": "MIT", "dependencies": { "has-flag": "^4.0.0" }, @@ -2062,35 +2362,58 @@ } }, "node_modules/tar-fs": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.4.tgz", - "integrity": "sha512-5AFQU8b9qLfZCX9zp2duONhPmZv0hGYiBPJsyUdqMjzq/mqVpy/rEUSeHk1+YitmxugaptgBh5oDGU3VsAJq4w==", + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.6.tgz", + "integrity": "sha512-iokBDQQkUyeXhgPYaZxmczGPhnhXZ0CmrqI+MOb/WFGS9DW5wnfrLgtjUJBvz50vQ3qfRwJ62QVoCFu8mPVu5w==", + "license": "MIT", "dependencies": { - "mkdirp-classic": "^0.5.2", "pump": "^3.0.0", "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^2.1.1", + "bare-path": "^2.1.0" + } + }, + "node_modules/tar-fs/node_modules/pump": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", + "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" } }, "node_modules/tar-stream": { "version": "3.1.7", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", + "license": "MIT", "dependencies": { "b4a": "^1.6.4", "fast-fifo": "^1.2.0", "streamx": "^2.15.0" } }, + "node_modules/text-decoder": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.1.tgz", + "integrity": "sha512-x9v3H/lTKIJKQQe7RPQkLfKAnc9lUTkWDypIQgTzPJAq+5/GCDHonmshfvlsNSj58yyshbIJJDLmU15qNERrXQ==", + "license": "Apache-2.0" + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/through2": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", + "license": "MIT", "dependencies": { "readable-stream": "~2.3.6", "xtend": "~4.0.1" @@ -2101,6 +2424,7 @@ "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", "dev": true, + "license": "MIT", "dependencies": { "is-number": "^7.0.0" }, @@ -2109,12 +2433,13 @@ } }, "node_modules/ts-api-utils": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.0.3.tgz", - "integrity": "sha512-wNMeqtMz5NtwpT/UZGY5alT+VoKdSsOOP/kqHFcUW1P/VRhH2wJ48+DN2WwUliNbQ976ETwDL0Ifd2VVvgonvg==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.4.0.tgz", + "integrity": "sha512-032cPxaEKwM+GT3vA5JXNzIaizx388rhsSW79vGRNGXfRRAdEAn2mvk36PvK5HnOchyWZ7afLEXqYCvPCrzuzQ==", "dev": true, + "license": "MIT", "engines": { - "node": ">=16.13.0" + "node": ">=16" }, "peerDependencies": { "typescript": ">=4.2.0" @@ -2125,6 +2450,7 @@ "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", "dev": true, + "license": "MIT", "dependencies": { "prelude-ls": "^1.2.1" }, @@ -2137,6 +2463,7 @@ "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", "dev": true, + "license": "(MIT OR CC0-1.0)", "engines": { "node": ">=10" }, @@ -2145,10 +2472,11 @@ } }, "node_modules/typescript": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz", - "integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==", + "version": "5.6.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.3.tgz", + "integrity": "sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==", "dev": true, + "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -2157,11 +2485,19 @@ "node": ">=14.17" } }, + "node_modules/undici-types": { + "version": "6.19.8", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", + "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", + "dev": true, + "license": "MIT" + }, "node_modules/uri-js": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", "dev": true, + "license": "BSD-2-Clause", "dependencies": { "punycode": "^2.1.0" } @@ -2169,13 +2505,15 @@ "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", "dev": true, + "license": "ISC", "dependencies": { "isexe": "^2.0.0" }, @@ -2186,17 +2524,29 @@ "node": ">= 8" } }, + "node_modules/word-wrap": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/workerpool": { "version": "6.5.1", "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.5.1.tgz", "integrity": "sha512-Fs4dNYcsdpYSAfVxhnl1L5zTksjvOJxtC5hzMNl+1t9B8hTJTdKDyZ5ju7ztgPy+ft9tBFXoOlDNiOT9WUXZlA==", - "dev": true + "dev": true, + "license": "Apache-2.0" }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", "dev": true, + "license": "MIT", "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", @@ -2212,12 +2562,14 @@ "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", "engines": { "node": ">=0.4" } @@ -2227,21 +2579,17 @@ "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", "dev": true, + "license": "ISC", "engines": { "node": ">=10" } }, - "node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true - }, "node_modules/yargs": { "version": "16.2.0", "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", "dev": true, + "license": "MIT", "dependencies": { "cliui": "^7.0.2", "escalade": "^3.1.1", @@ -2260,6 +2608,7 @@ "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", "dev": true, + "license": "ISC", "engines": { "node": ">=10" } @@ -2269,6 +2618,7 @@ "resolved": "https://registry.npmjs.org/yargs-unparser/-/yargs-unparser-2.0.0.tgz", "integrity": "sha512-7pRTIA9Qc1caZ0bZ6RYRGbHJthJWuakf+WmHK0rVeLkNrrGhfoabBNdue6kdINI6r4if7ocq9aD/n7xwKOdzOA==", "dev": true, + "license": "MIT", "dependencies": { "camelcase": "^6.0.0", "decamelize": "^4.0.0", @@ -2284,6 +2634,7 @@ "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", "dev": true, + "license": "MIT", "engines": { "node": ">=10" }, diff --git a/src/bindings/js/node/package.json b/src/bindings/js/node/package.json index 1ca1f10cdf57c2..10fc6d38bd51f4 100644 --- a/src/bindings/js/node/package.json +++ b/src/bindings/js/node/package.json @@ -1,6 +1,6 @@ { "name": "openvino-node", - "version": "2024.4.0", + "version": "2024.5.0-0", "description": "OpenVINO™ utils for using from Node.js environment", "repository": { "url": "git+https://github.com/openvinotoolkit/openvino.git", @@ -44,6 +44,7 @@ "tar-fs": "^3.0.4" }, "binary": { + "version": "2024.5.0", "module_path": "./bin/", "remote_path": "./repositories/openvino/nodejs_bindings/{version}/{platform}/", "package_name": "openvino_nodejs_bindings_{platform}_{version}_{arch}.tar.gz", diff --git a/src/bindings/js/node/scripts/download-runtime.js b/src/bindings/js/node/scripts/download-runtime.js index 90bece67161a6a..0fe2998953c4f3 100644 --- a/src/bindings/js/node/scripts/download-runtime.js +++ b/src/bindings/js/node/scripts/download-runtime.js @@ -17,7 +17,7 @@ async function main() { await BinaryManager.prepareBinary( join(__dirname, '..'), - packageJson.version, + packageJson.binary.version || packageJson.version, packageJson.binary, { force, ignoreIfExists, proxy }, ); diff --git a/src/bindings/python/src/pyopenvino/graph/ops/constant.cpp b/src/bindings/python/src/pyopenvino/graph/ops/constant.cpp index d5b7e5878e0184..20361b1fc5ca19 100644 --- a/src/bindings/python/src/pyopenvino/graph/ops/constant.cpp +++ b/src/bindings/python/src/pyopenvino/graph/ops/constant.cpp @@ -121,6 +121,22 @@ void regclass_graph_op_Constant(py::module m) { } }); + constant.def("get_tensor_view", + &ov::op::v0::Constant::get_tensor_view, + R"( + Get view on constant data as tensor. + + :rtype: openvino.Tensor + )"); + + constant.def("get_strides", + &ov::op::v0::Constant::get_strides, + R"( + Constant's strides in bytes. + + :rtype: openvino.Strides + )"); + // TODO: Remove in future and re-use `get_data` // Provide buffer access constant.def_buffer([](const ov::op::v0::Constant& self) -> py::buffer_info { @@ -236,6 +252,22 @@ void regclass_graph_op_Constant(py::module m) { :rtype: numpy.array )"); + constant.def_property_readonly("tensor_view", + &ov::op::v0::Constant::get_tensor_view, + R"( + Get view on constant data as tensor. + + :rtype: openvino.Tensor + )"); + + constant.def_property_readonly("strides", + &ov::op::v0::Constant::get_strides, + R"( + Constant's strides in bytes. + + :rtype: openvino.Strides + )"); + constant.def("__repr__", [](const ov::op::v0::Constant& self) { std::stringstream shapes_ss; for (size_t i = 0; i < self.get_output_size(); ++i) { diff --git a/src/bindings/python/src/pyopenvino/graph/strides.cpp b/src/bindings/python/src/pyopenvino/graph/strides.cpp index daecb6e9e3a312..951f4c3e687287 100644 --- a/src/bindings/python/src/pyopenvino/graph/strides.cpp +++ b/src/bindings/python/src/pyopenvino/graph/strides.cpp @@ -16,6 +16,14 @@ namespace py = pybind11; +template +bool compare_strides(const ov::Strides& a, const T& b) { + return a.size() == b.size() && + std::equal(a.begin(), a.end(), b.begin(), [](const size_t& elem_a, const py::handle& elem_b) { + return elem_a == elem_b.cast(); + }); +} + void regclass_graph_Strides(py::module m) { py::class_> strides(m, "Strides"); strides.doc() = "openvino.runtime.Strides wraps ov::Strides"; @@ -48,6 +56,27 @@ void regclass_graph_Strides(py::module m) { return self.size(); }); + strides.def( + "__eq__", + [](const ov::Strides& a, const ov::Strides& b) { + return a == b; + }, + py::is_operator()); + + strides.def( + "__eq__", + [](const ov::Strides& a, const py::tuple& b) { + return compare_strides(a, b); + }, + py::is_operator()); + + strides.def( + "__eq__", + [](const ov::Strides& a, const py::list& b) { + return compare_strides(a, b); + }, + py::is_operator()); + strides.def( "__iter__", [](const ov::Strides& self) { diff --git a/src/bindings/python/tests/test_graph/test_constant.py b/src/bindings/python/tests/test_graph/test_constant.py index 7b349ad7cd94b1..a8b4dfa3e1a26b 100644 --- a/src/bindings/python/tests/test_graph/test_constant.py +++ b/src/bindings/python/tests/test_graph/test_constant.py @@ -6,7 +6,7 @@ import openvino as ov import openvino.runtime.opset13 as ops -from openvino import Type, PartialShape, Model, Tensor, compile_model +from openvino import Type, PartialShape, Model, Strides, Tensor, compile_model from openvino.runtime.op import Constant from openvino.helpers import pack_data, unpack_data @@ -756,7 +756,7 @@ def test_get_data_casting_packed(src_dtype, ov_type, dst_dtype, copy_flag): ], ) def test_const_from_tensor(shared_flag): - shape = [1, 3, 32, 32] + shape = [1, 2, 3, 3] arr = np.ones(shape).astype(np.float32) ov_tensor = Tensor(arr, shape, Type.f32) ov_const = ops.constant(tensor=ov_tensor, shared_memory=shared_flag) @@ -771,3 +771,6 @@ def test_const_from_tensor(shared_flag): else: assert not np.array_equal(ov_const.data, arr) assert not np.shares_memory(arr, ov_const.data) + + assert ov_const.strides == [72, 36, 12, 4] + assert ov_const.get_tensor_view().get_strides() == Strides([72, 36, 12, 4]) diff --git a/src/common/transformations/include/ov_ops/rotary_positional_embeddings.hpp b/src/common/transformations/include/ov_ops/rotary_positional_embeddings.hpp index dcb9aef187d2d9..08c1aa8e3f5ad8 100644 --- a/src/common/transformations/include/ov_ops/rotary_positional_embeddings.hpp +++ b/src/common/transformations/include/ov_ops/rotary_positional_embeddings.hpp @@ -23,13 +23,14 @@ class TRANSFORMATIONS_API RoPE : public Op { struct Config { size_t slice_start = 0; // slice inner-most dimensions of input size_t slice_stop = 0; - bool input_trans0213 = false; // transpose input dim 1&2 - bool is_interleaved = false; // interleaved mode, implies trans0213 happens after RoPE - size_t rotary_ndims = 0; // dimensions to be embedded (d in the description) - bool is_chatglm = false; // chatglm is special which overrides other setting - bool support_2d_rope = false; // 2d rope mode, Support 2 dimentional rope which is independant of batch and - // each head. change input order to [batch, head_cnt, 4608] to support 2d rope - bool is_qwen = false; // Qwen is special which overrides other setting + bool input_trans0213 = false; // transpose input dim 1&2 + bool output_trans0213 = false; // implies trans0213 happens after RoPE + bool is_interleaved = false; // coordinates are interleaved + size_t rotary_ndims = 0; // dimensions to be embedded (d in the description) + bool is_chatglm = false; // chatglm is special which overrides other setting + bool support_2d_rope = false; // 2d rope mode, Support 2 dimentional rope which is independant of batch and + // each head. change input order to [batch, head_cnt, 4608] to support 2d rope + bool is_qwen = false; // Qwen is special which overrides other setting size_t head_cnt = 0; size_t head_size = 0; int gather_position_arg_id = diff --git a/src/common/transformations/include/transformations/common_optimizations/fuse_rotary_positional_embeddings.hpp b/src/common/transformations/include/transformations/common_optimizations/fuse_rotary_positional_embeddings.hpp index eb1c92bcf9607f..3449151ab93ac5 100644 --- a/src/common/transformations/include/transformations/common_optimizations/fuse_rotary_positional_embeddings.hpp +++ b/src/common/transformations/include/transformations/common_optimizations/fuse_rotary_positional_embeddings.hpp @@ -12,6 +12,7 @@ namespace pass { class TRANSFORMATIONS_API RoPEFusion; class TRANSFORMATIONS_API RoPEFusionGPTNEOX; +class TRANSFORMATIONS_API RoPEFusionFlux; class TRANSFORMATIONS_API RoPEFusionGPTJ; class TRANSFORMATIONS_API RoPEFusionChatGLM; class TRANSFORMATIONS_API RoPEFusionQwen; @@ -29,6 +30,12 @@ class ov::pass::RoPEFusionGPTNEOX : public ov::pass::MatcherPass { RoPEFusionGPTNEOX(); }; +class ov::pass::RoPEFusionFlux : public ov::pass::MatcherPass { +public: + OPENVINO_RTTI("RoPEFusionFlux", "0"); + RoPEFusionFlux(); +}; + class ov::pass::RoPEFusionGPTJ : public ov::pass::MatcherPass { public: OPENVINO_RTTI("RoPEFusionGPTJ", "0"); @@ -85,6 +92,7 @@ class ov::pass::RoPEFusion : public ov::pass::GraphRewrite { public: OPENVINO_RTTI("RoPEFusion", "0"); RoPEFusion(bool support_2d_rope = false) { + add_matcher(); add_matcher(); add_matcher(); // optional heads & tails are fused in separate matcher pass, diff --git a/src/common/transformations/include/transformations/op_conversions/convert_reduce_to_pooling.hpp b/src/common/transformations/include/transformations/op_conversions/convert_reduce_to_pooling.hpp index b74a0ff538e011..662660b926aa52 100644 --- a/src/common/transformations/include/transformations/op_conversions/convert_reduce_to_pooling.hpp +++ b/src/common/transformations/include/transformations/op_conversions/convert_reduce_to_pooling.hpp @@ -72,7 +72,7 @@ ov::matcher_pass_callback ConvertReduceBase::convert_reduce_to_pooling() { return [&](ov::pass::pattern::Matcher& m) { auto reduce = std::dynamic_pointer_cast(m.get_match_root()); - if (!reduce || transformation_callback(reduce)) { + if (!reduce || transformation_callback(reduce) || ov::shape_size(reduce->input_value(0).get_shape()) == 0) { return false; } diff --git a/src/common/transformations/src/ov_ops/rotary_positional_embeddings.cpp b/src/common/transformations/src/ov_ops/rotary_positional_embeddings.cpp index 3e75e2b88df266..88a42a7f456db1 100644 --- a/src/common/transformations/src/ov_ops/rotary_positional_embeddings.cpp +++ b/src/common/transformations/src/ov_ops/rotary_positional_embeddings.cpp @@ -76,7 +76,7 @@ void RoPE::validate_and_infer_types() { if (m_config.input_trans0213) { // transpose 0213 ([B,L,H,S]=>[B,H,L,S]) happens before RoPE std::swap(input_pshape[2], input_pshape[1]); - } else if (m_config.is_interleaved) { + } else if (m_config.output_trans0213) { // transpose 0213 ([B,L,H,S]=>[B,H,L,S]) happens after RoPE std::swap(input_pshape[2], input_pshape[1]); } @@ -90,6 +90,7 @@ bool RoPE::visit_attributes(ov::AttributeVisitor& visitor) { visitor.on_attribute("slice_start", m_config.slice_start); visitor.on_attribute("slice_stop", m_config.slice_stop); visitor.on_attribute("input_trans0213", m_config.input_trans0213); + visitor.on_attribute("output_trans0213", m_config.output_trans0213); visitor.on_attribute("is_interleaved", m_config.is_interleaved); visitor.on_attribute("rotary_ndims", m_config.rotary_ndims); visitor.on_attribute("is_chatglm", m_config.is_chatglm); diff --git a/src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp b/src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp index f002e0043a8744..ec49dd7152fed1 100644 --- a/src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/fuse_rotary_positional_embeddings.cpp @@ -23,6 +23,94 @@ using namespace ov::gen_pattern; +ov::pass::RoPEFusionFlux::RoPEFusionFlux() { + MATCHER_SCOPE(RoPEFusionFlux); + // x[?,24,?,128] + // x1 = reshape(x, [?,24,?,64,2]) + // x1_0, x1_1 = split(x1, -1) + // x2 = concat(x1_0, x1_1 * (-1), -1) + // x3 = reshape(x2, [?,24,?,128]) + // y1 = x * t_cos + // y2 = x3 * t_sin + // y = y1 + y2 + auto x = makePattern(ov::Rank(4)); + auto t_cos = makePattern(ov::Rank(4)); + auto t_sin = makePattern(ov::Rank(4)); + + auto num_heads = ov::gen_pattern::Symbol("num_heads"); + auto head_size = ov::gen_pattern::Symbol("head_size"); + + auto x1_target_shape = makeConst({0, num_heads, 0, -1, 2}); + auto x1 = makePattern({x, x1_target_shape}, {{"special_zero", true}}); + auto split = makePattern({x1, -1}, {{"num_splits", 2}}); + split->set_output_size(2); + + // 3 versions of mulitply by -1 depending on transformations execution prior to this pass + auto x1_1_neg_1 = makePattern({split->output(1), -1.0f}, {{"auto_broadcast", "numpy"}}); + + auto squeeze_2 = makePattern({split->output(1), -1}); + auto x1_1_neg_2 = makePattern({squeeze_2, -1.0f}, {{"auto_broadcast", "numpy"}}); + auto unsqueeze_2 = makePattern({x1_1_neg_2, -1}); + + auto x1_1_neg_3 = makePattern({split->output(1), -1.0f}, {{"auto_broadcast", "numpy"}}); + auto squeeze_3 = makePattern({x1_1_neg_3, -1}); + auto unsqueeze_3 = makePattern({squeeze_3, -1}); + + auto x2 = makePattern({x1_1_neg_1 | unsqueeze_2 | unsqueeze_3, split->output(0)}, {{"axis", -1}}); + auto x3_target_shape = makeConst({0, num_heads, 0, head_size}); + auto x3 = makePattern({x2, x3_target_shape}, {{"special_zero", true}}); + + auto y1 = makePattern({x, t_cos}, {{"auto_broadcast", "numpy"}}); + auto y2 = makePattern({x3, t_sin}, {{"auto_broadcast", "numpy"}}); + + auto y = makePattern({y1, y2}, {{"auto_broadcast", "numpy"}}); + auto result = y; + + matcher_pass_callback callback = [OV_CAPTURE_CPY_AND_THIS](ov::pass::pattern::Matcher& m) { + PatternValidator validator(m); + if (!validator) { + return false; + } + + const auto& pattern_map = m.get_pattern_value_map(); + auto root = m.get_match_root(); + + op::internal::RoPE::Config config; + config.head_cnt = static_cast(validator["num_heads"]); + config.head_size = static_cast(validator["head_size"]); + config.rotary_ndims = config.head_size; + config.is_interleaved = true; + config.output_trans0213 = false; + + OutputVector new_args; + new_args.push_back(pattern_map.at(x)); + new_args.push_back(pattern_map.at(t_cos)); + new_args.push_back(pattern_map.at(t_sin)); + + auto old_node = root; + auto new_node = std::make_shared(new_args, config); + new_node->set_friendly_name(old_node->get_friendly_name()); + ov::copy_runtime_info({pattern_map.at(x1).get_node_shared_ptr(), + pattern_map.at(split).get_node_shared_ptr(), + pattern_map.at(x2).get_node_shared_ptr(), + pattern_map.at(x3).get_node_shared_ptr(), + pattern_map.at(y1).get_node_shared_ptr(), + pattern_map.at(y2).get_node_shared_ptr(), + pattern_map.at(result).get_node_shared_ptr()}, + new_node); + + ov::replace_node(old_node, new_node); + + // this new node may match following additional matchers + register_new_node(new_node); + + return true; + }; + + auto m = std::make_shared(result, matcher_name); + this->register_matcher(m, callback); +} + ov::pass::RoPEFusionGPTNEOX::RoPEFusionGPTNEOX() { MATCHER_SCOPE(RoPEFusionGPTNEOX); @@ -373,6 +461,7 @@ ov::pass::RoPEFusionGPTJ::RoPEFusionGPTJ() { OutputVector new_args; config.rotary_ndims = static_cast(validator["ndims"]); + config.output_trans0213 = true; config.is_interleaved = true; // input is [B,L,H,S] diff --git a/src/common/transformations/src/transformations/common_optimizations/remove_multi_subgraph_op_dangling_params.cpp b/src/common/transformations/src/transformations/common_optimizations/remove_multi_subgraph_op_dangling_params.cpp index d86b4b71f102c7..fed6eaf9710420 100644 --- a/src/common/transformations/src/transformations/common_optimizations/remove_multi_subgraph_op_dangling_params.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/remove_multi_subgraph_op_dangling_params.cpp @@ -11,10 +11,30 @@ #include "openvino/op/tensor_iterator.hpp" #include "openvino/op/util/multi_subgraph_base.hpp" #include "openvino/pass/pattern/op/wrap_type.hpp" +#include "openvino/util/common_util.hpp" #include "transformations/utils/utils.hpp" using namespace ov::op::util; +namespace { +/** @brief Value to mark that input idx has been removed (at least one removed so last idx will be always available) */ +constexpr auto mark_removed = std::numeric_limits::max(); + +constexpr bool is_not_removed_idx(const decltype(mark_removed) idx) { + return mark_removed != idx; +} + +uint64_t get_updated_idx(uint64_t idx, uint64_t removed_idx) { + if (idx == removed_idx) { + return mark_removed; + } else if (is_not_removed_idx(idx) && idx > removed_idx) { + return idx - 1; + } else { + return idx; + } +}; +} // namespace + bool ov::pass::RemoveMultiSubGraphOpDanglingParamsResults::run_on_model(const std::shared_ptr& m) { RUN_ON_MODEL_SCOPE(RemoveMultiSubGraphOpDanglingParamsResults); bool is_changed = false; @@ -117,7 +137,6 @@ bool ov::pass::RemoveMultiSubGraphOpDanglingParamsResults::run_on_model(const st // Remove inputs bool pass_required = false; std::set required_inputs_indices; - auto op_inputs = multi_subgraph_op->input_values(); std::vector> to_remove_descriptors_indexes; to_remove_descriptors_indexes.resize(subgraphs_size); for (size_t body_idx = 0; body_idx < subgraphs_size; ++body_idx) { @@ -142,64 +161,57 @@ bool ov::pass::RemoveMultiSubGraphOpDanglingParamsResults::run_on_model(const st using DescType = op::util::MultiSubGraphOp::MultiSubgraphInputDescriptionVector; auto update_body_param_desc = [](DescType& descriptors, uint64_t removed_body_idx) { for (auto& desc : descriptors) { - if (desc->m_body_parameter_index > removed_body_idx) { - desc->m_body_parameter_index--; - } + desc->m_body_parameter_index = get_updated_idx(desc->m_body_parameter_index, removed_body_idx); } }; auto update_op_inputs_desc = [&subgraphs_size](const std::shared_ptr& op, - std::set& required_inputs_indices, uint64_t removed_loop_idx) { - std::set new_required_inputs_indices; for (size_t body_idx = 0; body_idx < subgraphs_size; ++body_idx) { auto& descriptors = op->get_input_descriptions(static_cast(body_idx)); for (auto& desc : descriptors) { - if (desc->m_input_index > removed_loop_idx) { - desc->m_input_index--; - } + desc->m_input_index = get_updated_idx(desc->m_input_index, removed_loop_idx); } } - for (auto input_index : required_inputs_indices) { - if (input_index > removed_loop_idx) { - new_required_inputs_indices.insert(input_index - 1); - } else { - new_required_inputs_indices.insert(input_index); - } + }; + + const auto update_required_input_indicies = [](std::set& required_inputs_indices, + uint64_t removed_input_idx) { + std::set new_required_inputs_indices; + for (const auto& input_index : required_inputs_indices) { + new_required_inputs_indices.insert(input_index > removed_input_idx ? input_index - 1 : input_index); } - required_inputs_indices = new_required_inputs_indices; + required_inputs_indices = std::move(new_required_inputs_indices); }; // Remove dangling body params and input and update input descriptors + auto op_inputs = multi_subgraph_op->input_values(); for (size_t body_idx = 0; body_idx < subgraphs_size; ++body_idx) { auto& body_in_descriptors = multi_subgraph_op->get_input_descriptions(static_cast(body_idx)); - auto& body_func = multi_subgraph_op->get_function(static_cast(body_idx)); - auto& body_params = body_func->get_parameters(); op::util::MultiSubGraphOp::MultiSubgraphInputDescriptionVector updated_body_in_descriptors; + for (size_t desc_idx = 0; desc_idx < body_in_descriptors.size(); ++desc_idx) { - if (std::count(std::begin(to_remove_descriptors_indexes[body_idx]), - std::end(to_remove_descriptors_indexes[body_idx]), - desc_idx) > 0) { - if (body_in_descriptors[desc_idx]->m_body_parameter_index < body_params.size()) { - auto& body_param = body_params[body_in_descriptors[desc_idx]->m_body_parameter_index]; - body_func->remove_parameter(body_param); - // Move all body indexes which are after these indicated by to_remove_descriptors_indexes - update_body_param_desc(body_in_descriptors, - body_in_descriptors[desc_idx]->m_body_parameter_index); - } - // remove dangling input of MultiSubGraphOp which was not removed earlier - auto current_input_idx = body_in_descriptors[desc_idx]->m_input_index; - // the same input tensor can go to different input ports - if (current_input_idx < op_inputs.size() && - std::count(std::begin(required_inputs_indices), - std::end(required_inputs_indices), - current_input_idx) == 0 && - std::count(std::begin(op_inputs), std::end(op_inputs), op_inputs[current_input_idx]) > 0) { - op_inputs.erase(std::next(op_inputs.begin(), current_input_idx)); - // Move all input indexes (in all bodies) which are after these indicated by - // to_remove_descriptors_indexes and are not used in any body - update_op_inputs_desc(multi_subgraph_op, required_inputs_indices, current_input_idx); - } - } else { - updated_body_in_descriptors.emplace_back(body_in_descriptors[desc_idx]); + auto& current_body_desc = body_in_descriptors[desc_idx]; + const auto current_body_parameter_idx = current_body_desc->m_body_parameter_index; + if (!util::contains(to_remove_descriptors_indexes[body_idx], desc_idx)) { + updated_body_in_descriptors.emplace_back(current_body_desc); + } else if (is_not_removed_idx(current_body_parameter_idx)) { + auto& body_func = multi_subgraph_op->get_function(body_idx); + const auto& body_params = body_func->get_parameters(); + + body_func->remove_parameter(body_params[current_body_parameter_idx]); + // Move all body indexes which are after these indicated by to_remove_descriptors_indexes + update_body_param_desc(body_in_descriptors, current_body_parameter_idx); + } + + const auto current_input_idx = current_body_desc->m_input_index; + // remove dangling input of MultiSubGraphOp which was not removed earlier + // the same input tensor can go to different input ports + if (!util::contains(required_inputs_indices, current_input_idx) && + is_not_removed_idx(current_input_idx)) { + op_inputs.erase(op_inputs.begin() + current_input_idx); + // Move all input indexes (in all bodies) which are after these indicated by + // to_remove_descriptors_indexes and are not used in any body + update_op_inputs_desc(multi_subgraph_op, current_input_idx); + update_required_input_indicies(required_inputs_indices, current_input_idx); } } multi_subgraph_op->set_input_descriptions(static_cast(body_idx), updated_body_in_descriptors); diff --git a/src/common/transformations/src/transformations/common_optimizations/reverse_shape_and_type_infer.cpp b/src/common/transformations/src/transformations/common_optimizations/reverse_shape_and_type_infer.cpp index 9a06201f688675..9fbaed822cdfba 100644 --- a/src/common/transformations/src/transformations/common_optimizations/reverse_shape_and_type_infer.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/reverse_shape_and_type_infer.cpp @@ -23,6 +23,25 @@ #include "openvino/op/util/unary_elementwise_arithmetic.hpp" #include "transformations/utils/utils.hpp" +namespace { + +void set_source_output_type_shape(const ov::Node& node, + const ov::element::Type& et, + const ov::PartialShape& new_shape, + const size_t port) { + const auto source_output = node.get_input_source_output(port); + source_output.get_node()->set_output_type(source_output.get_index(), et, new_shape); +} + +void set_source_output_shape(const ov::Node& node, const ov::PartialShape& new_shape, const size_t port) { + set_source_output_type_shape(node, node.get_input_element_type(port), new_shape, port); +} + +void set_source_output_type(const ov::Node& node, const ov::element::Type& et, const size_t port) { + set_source_output_type_shape(node, et, node.get_input_partial_shape(port), port); +} +} // namespace + bool ov::pass::ReverseShapeAndTypeInfer::inherit_output_shape(const std::shared_ptr& node, const std::vector& input_idxs) { auto is_changed = false; @@ -30,7 +49,9 @@ bool ov::pass::ReverseShapeAndTypeInfer::inherit_output_shape(const std::shared_ for (auto idx : input_idxs) { if (idx < node->get_input_size() && node->get_input_partial_shape(idx).compatible(output_shape)) { - PartialShape::merge_into(node->get_input_tensor(idx).m_partial_shape, output_shape); + auto new_shape = node->get_input_partial_shape(idx); + PartialShape::merge_into(new_shape, output_shape); + set_source_output_shape(*node, new_shape, idx); is_changed = true; } } @@ -44,7 +65,7 @@ bool ov::pass::ReverseShapeAndTypeInfer::inherit_output_rank(const std::shared_p for (auto idx : input_idxs) { if (idx < node->get_input_size() && node->get_input_partial_shape(idx).rank().is_dynamic()) { - node->get_input_tensor(idx).m_partial_shape = ov::PartialShape::dynamic(output_shape.rank()); + set_source_output_shape(*node, ov::PartialShape::dynamic(output_shape.rank()), idx); is_changed = true; } } @@ -58,7 +79,7 @@ bool ov::pass::ReverseShapeAndTypeInfer::inherit_output_type(const std::shared_p for (auto idx : input_idxs) { if (idx < node->get_input_size() && node->get_input_element_type(idx).is_dynamic()) { - node->get_input_tensor(idx).m_element_type = output_type; + set_source_output_type(*node, output_type, idx); is_changed = true; } } @@ -92,7 +113,9 @@ bool ov::pass::ReverseShapeAndTypeInfer::run_on_model(const std::shared_ptrget_input_partial_shape(1); if (weigths_pshape.rank().is_static() && op->get_input_partial_shape(1).rank().is_static() && weigths_pshape[1] != 1) { - op->get_input_tensor(0).m_partial_shape[1] = weigths_pshape[1]; + auto new_shape = op->get_input_partial_shape(0); + new_shape[1] = weigths_pshape[1]; + set_source_output_shape(*op, new_shape, 0); } is_changed |= inherit_output_type(op, {0, 1}); } else if (ov::as_type_ptr(op)) { @@ -101,7 +124,9 @@ bool ov::pass::ReverseShapeAndTypeInfer::run_on_model(const std::shared_ptrget_input_partial_shape(1); if (weigths_pshape.rank().is_static() && op->get_input_partial_shape(1).rank().is_static() && weigths_pshape[2] != 1) { - op->get_input_tensor(0).m_partial_shape[1] = weigths_pshape[0] * weigths_pshape[2]; + auto new_shape = op->get_input_partial_shape(0); + new_shape[1] = weigths_pshape[0] * weigths_pshape[2]; + set_source_output_shape(*op, new_shape, 0); } is_changed |= inherit_output_type(op, {0, 1}); } else if (ov::as_type_ptr(op)) { @@ -110,7 +135,9 @@ bool ov::pass::ReverseShapeAndTypeInfer::run_on_model(const std::shared_ptrget_input_partial_shape(1); if (weigths_pshape.rank().is_static() && op->get_input_partial_shape(1).rank().is_static() && weigths_pshape[0] != 1) { - op->get_input_tensor(0).m_partial_shape[1] = weigths_pshape[0]; + auto new_shape = op->get_input_partial_shape(0); + new_shape[1] = weigths_pshape[0]; + set_source_output_shape(*op, new_shape, 0); } is_changed |= inherit_output_type(op, {0, 1}); } else if (ov::as_type_ptr(op)) { @@ -119,7 +146,9 @@ bool ov::pass::ReverseShapeAndTypeInfer::run_on_model(const std::shared_ptrget_input_partial_shape(1); if (weigths_pshape.rank().is_static() && op->get_input_partial_shape(1).rank().is_static() && weigths_pshape[1] != 1) { - op->get_input_tensor(0).m_partial_shape[1] = weigths_pshape[0] * weigths_pshape[1]; + auto new_shape = op->get_input_partial_shape(0); + new_shape[1] = weigths_pshape[0] * weigths_pshape[1]; + set_source_output_shape(*op, new_shape, 0); } is_changed |= inherit_output_type(op, {0, 1}); } else if (ov::as_type_ptr(op)) { @@ -131,10 +160,10 @@ bool ov::pass::ReverseShapeAndTypeInfer::run_on_model(const std::shared_ptrget_input_partial_shape(1); auto pads_end_shape = op->get_input_partial_shape(2); if (pads_begin_shape.is_static() && pads_begin_shape.size() > 0) { - op->get_input_tensor(0).m_partial_shape = PartialShape::dynamic(pads_begin_shape[0]); + set_source_output_shape(*op, PartialShape::dynamic(pads_begin_shape[0]), 0); is_changed = true; } else if (pads_end_shape.is_static() && pads_end_shape.size() > 0) { - op->get_input_tensor(0).m_partial_shape = PartialShape::dynamic(pads_end_shape[0]); + set_source_output_shape(*op, PartialShape::dynamic(pads_end_shape[0]), 0); is_changed = true; } } @@ -147,15 +176,17 @@ bool ov::pass::ReverseShapeAndTypeInfer::run_on_model(const std::shared_ptrget_input_partial_shape(0).rank(); auto in1_rank = op->get_input_partial_shape(1).rank(); if (in0_rank.is_dynamic() && in1_rank.is_static()) { - if (eltwise->get_autob() == ov::op::AutoBroadcastType::NONE) - op->get_input_tensor(0).m_partial_shape = output_shape; - else if (in1_rank.get_length() < output_shape.rank().get_length()) - op->get_input_tensor(0).m_partial_shape = PartialShape::dynamic(output_shape.rank()); + if (eltwise->get_autob() == ov::op::AutoBroadcastType::NONE) { + set_source_output_shape(*op, output_shape, 0); + } else if (in1_rank.get_length() < output_shape.rank().get_length()) { + set_source_output_shape(*op, PartialShape::dynamic(output_shape.rank()), 0); + } } else if (in1_rank.is_dynamic() && in0_rank.is_static()) { - if (eltwise->get_autob() == ov::op::AutoBroadcastType::NONE) - op->get_input_tensor(1).m_partial_shape = output_shape; - else if (in0_rank.get_length() < output_shape.rank().get_length()) - op->get_input_tensor(1).m_partial_shape = PartialShape::dynamic(output_shape.rank()); + if (eltwise->get_autob() == ov::op::AutoBroadcastType::NONE) { + set_source_output_shape(*op, output_shape, 1); + } else if (in0_rank.get_length() < output_shape.rank().get_length()) { + set_source_output_shape(*op, PartialShape::dynamic(output_shape.rank()), 1); + } } } is_changed |= inherit_output_type(op, {0, 1}); @@ -172,7 +203,9 @@ bool ov::pass::ReverseShapeAndTypeInfer::run_on_model(const std::shared_ptrget_input_size() && op->get_input_partial_shape(idx).compatible(input_pshape)) { - PartialShape::merge_into(op->get_input_tensor(idx).m_partial_shape, input_pshape); + auto new_shape = op->get_input_partial_shape(idx); + PartialShape::merge_into(new_shape, input_pshape); + set_source_output_shape(*op, new_shape, idx); is_changed = true; } } @@ -189,8 +222,9 @@ bool ov::pass::ReverseShapeAndTypeInfer::run_on_model(const std::shared_ptrget_input_partial_shape(1); if (in1_pshape.is_static()) { auto num_dims = in1_pshape.size() == 0 ? 1 : in1_pshape[0].get_length(); - op->get_input_tensor(0).m_partial_shape = - PartialShape::dynamic(output_shape.rank().get_length() + num_dims); + set_source_output_shape(*op, + PartialShape::dynamic(output_shape.rank().get_length() + num_dims), + 0); } } else if (in0_rank.is_static() && op->get_input_size() == 1) { // attempt to create second input @@ -215,8 +249,7 @@ bool ov::pass::ReverseShapeAndTypeInfer::run_on_model(const std::shared_ptrget_input_partial_shape(1); if (output_shape.rank().is_static() && in0_rank.is_dynamic() && in1_pshape.is_static()) { auto num_dims = in1_pshape.size() == 0 ? 1 : in1_pshape[0].get_length(); - op->get_input_tensor(0).m_partial_shape = - PartialShape::dynamic(output_shape.rank().get_length() - num_dims); + set_source_output_shape(*op, PartialShape::dynamic(output_shape.rank().get_length() - num_dims), 0); } is_changed |= inherit_output_type(op, {0}); } else if (const auto& if_op = ov::as_type_ptr(op)) { @@ -227,21 +260,24 @@ bool ov::pass::ReverseShapeAndTypeInfer::run_on_model(const std::shared_ptrget_results(); const auto& then_out_desc = if_op->get_output_descriptions(ov::op::v8::If::THEN_BODY_INDEX); const auto& else_out_desc = if_op->get_output_descriptions(ov::op::v8::If::ELSE_BODY_INDEX); + for (const auto& out_desc : then_out_desc) { const auto& out_indx = out_desc->m_output_index; const auto& body_indx = out_desc->m_body_value_index; - then_body_results[body_indx]->get_input_tensor(0).m_partial_shape = - if_op->get_output_partial_shape(out_indx); - then_body_results[body_indx]->get_input_tensor(0).m_element_type = - if_op->get_output_element_type(out_indx); + + set_source_output_type_shape(*then_body_results[body_indx], + if_op->get_output_element_type(out_indx), + if_op->get_output_partial_shape(out_indx), + 0); } + for (const auto& out_desc : else_out_desc) { const auto& out_indx = out_desc->m_output_index; const auto& body_indx = out_desc->m_body_value_index; - else_body_results[body_indx]->get_input_tensor(0).m_partial_shape = - if_op->get_output_partial_shape(out_indx); - else_body_results[body_indx]->get_input_tensor(0).m_element_type = - if_op->get_output_element_type(out_indx); + set_source_output_type_shape(*else_body_results[body_indx], + if_op->get_output_element_type(out_indx), + if_op->get_output_partial_shape(out_indx), + 0); } is_changed |= run_on_model(then_body); is_changed |= run_on_model(else_body); @@ -252,34 +288,30 @@ bool ov::pass::ReverseShapeAndTypeInfer::run_on_model(const std::shared_ptrm_input_index; const auto& body_indx = in_desc->m_body_parameter_index; - if (if_op->get_input_tensor(in_indx).get_partial_shape().rank().is_dynamic()) { - if_op->get_input_tensor(in_indx).m_partial_shape = - then_body_params.at(body_indx)->get_partial_shape(); - is_changed = true; - } - if (if_op->get_input_tensor(in_indx).get_element_type().is_dynamic()) { - if_op->get_input_tensor(in_indx).m_element_type = - then_body_params.at(body_indx)->get_element_type(); + if (if_op->get_input_tensor(in_indx).get_partial_shape().rank().is_dynamic() || + if_op->get_input_tensor(in_indx).get_element_type().is_dynamic()) { + set_source_output_type_shape(*if_op, + then_body_params.at(body_indx)->get_element_type(), + then_body_params.at(body_indx)->get_partial_shape(), + in_indx); is_changed = true; } } for (const auto& in_desc : else_in_desc) { const auto& in_indx = in_desc->m_input_index; const auto& body_indx = in_desc->m_body_parameter_index; - if (if_op->get_input_tensor(in_indx).get_partial_shape().rank().is_dynamic()) { - if_op->get_input_tensor(in_indx).m_partial_shape = - else_body_params.at(body_indx)->get_partial_shape(); - is_changed = true; - } - if (if_op->get_input_tensor(in_indx).get_element_type().is_dynamic()) { - if_op->get_input_tensor(in_indx).m_element_type = - else_body_params.at(body_indx)->get_element_type(); + if (if_op->get_input_tensor(in_indx).get_partial_shape().rank().is_dynamic() || + if_op->get_input_tensor(in_indx).get_element_type().is_dynamic()) { + set_source_output_type_shape(*if_op, + then_body_params.at(body_indx)->get_element_type(), + then_body_params.at(body_indx)->get_partial_shape(), + in_indx); is_changed = true; } } // Set type for If condition if (if_op->get_input_element_type(0).is_dynamic()) { - if_op->get_input_tensor(0).m_element_type = element::boolean; + set_source_output_type(*if_op, element::boolean, 0); is_changed = true; } @@ -288,7 +320,7 @@ bool ov::pass::ReverseShapeAndTypeInfer::run_on_model(const std::shared_ptrget_rt_info().count("tf_switch_merge_if") && if_op->get_rt_info()["tf_switch_merge_if"].as() && if_op->input_value(0).get_partial_shape().rank().is_dynamic()) { - if_op->get_input_tensor(0).m_partial_shape = ov::PartialShape({}); + set_source_output_shape(*if_op, PartialShape{}, 0); is_changed = true; } } else if (ov::as_type_ptr(op)) { @@ -301,24 +333,24 @@ bool ov::pass::ReverseShapeAndTypeInfer::run_on_model(const std::shared_ptrget_input_tensor(0).m_partial_shape, - PartialShape::dynamic(output_shape.rank())); + auto new_shape = op->get_input_partial_shape(0); + PartialShape::merge_into(new_shape, PartialShape::dynamic(output_shape.rank())); auto order_value = transpose_order->cast_vector(); OPENVINO_ASSERT(order_value.size() == static_cast(rank_length), "The length of Transpose order and the input rank mismatch"); for (int64_t dim_idx = 0; dim_idx < rank_length; ++dim_idx) { OPENVINO_ASSERT(0 <= order_value[dim_idx] && order_value[dim_idx] < rank_length, "Transpose order is out-of-range"); - op->get_input_tensor(0).m_partial_shape[order_value[dim_idx]] = output_shape[dim_idx]; + new_shape[order_value[dim_idx]] = output_shape[dim_idx]; } + set_source_output_shape(*op, new_shape, 0); is_changed = true; } else { is_changed |= inherit_output_rank(op, {0}); } } else if (transpose_order) { auto order_value = transpose_order->cast_vector(); - PartialShape::merge_into(op->get_input_tensor(0).m_partial_shape, - PartialShape::dynamic(order_value.size())); + set_source_output_shape(*op, PartialShape::dynamic(order_value.size()), 0); is_changed = true; } is_changed |= inherit_output_type(op, {0}); diff --git a/src/common/transformations/tests/common_optimizations/fuse_rotary_positional_embeddings.cpp b/src/common/transformations/tests/common_optimizations/fuse_rotary_positional_embeddings.cpp index ea928de5c01702..a42e11120d7276 100644 --- a/src/common/transformations/tests/common_optimizations/fuse_rotary_positional_embeddings.cpp +++ b/src/common/transformations/tests/common_optimizations/fuse_rotary_positional_embeddings.cpp @@ -6,7 +6,9 @@ #include +#include "common_test_utils/graph_comparator.hpp" #include "common_test_utils/ov_test_utils.hpp" +#include "openvino/core/node_vector.hpp" #include "openvino/opsets/opset1.hpp" #include "openvino/opsets/opset3.hpp" #include "ov_ops/rotary_positional_embeddings.hpp" @@ -133,6 +135,7 @@ TEST_F(TransformationTestsF, ConvertToROPE_LLama2_no_gather) { {{"config.slice_start", 0}, {"config.slice_stop", 0}, {"config.input_trans0213", true}, + {"config.output_trans0213", false}, {"config.is_interleaved", false}, {"config.is_chatglm", false}, {"config.support_2d_rope", false}, @@ -169,6 +172,7 @@ TEST_F(TransformationTestsF, ConvertToROPE_LLama2_with_gather) { {{"config.slice_start", 0}, {"config.slice_stop", 0}, {"config.input_trans0213", true}, + {"config.output_trans0213", false}, {"config.is_interleaved", false}, {"config.is_chatglm", false}, {"config.support_2d_rope", false}, @@ -308,6 +312,7 @@ TEST_F(TransformationTestsF, ConvertToROPE_GPTNEOX_no_gather) { {{"config.slice_start", 0}, {"config.slice_stop", ndims}, {"config.input_trans0213", true}, + {"config.output_trans0213", false}, {"config.is_interleaved", false}, {"config.is_chatglm", false}, {"config.support_2d_rope", false}, @@ -343,6 +348,7 @@ TEST_F(TransformationTestsF, ConvertToROPE_GPTNEOX_with_gather) { {{"config.slice_start", 0}, {"config.slice_stop", ndims}, {"config.input_trans0213", true}, + {"config.output_trans0213", false}, {"config.is_interleaved", false}, {"config.is_chatglm", false}, {"config.support_2d_rope", false}, @@ -459,6 +465,7 @@ TEST_F(TransformationTestsF, ConvertToROPE_GPTJ) { {{"config.slice_start", 0}, {"config.slice_stop", 0}, {"config.input_trans0213", false}, + {"config.output_trans0213", true}, {"config.is_interleaved", true}, {"config.is_chatglm", false}, {"config.support_2d_rope", false}, @@ -568,6 +575,7 @@ TEST_F(TransformationTestsF, ConvertToROPE_chatGML) { {{"config.slice_start", 0}, {"config.slice_stop", 4096}, {"config.input_trans0213", false}, + {"config.output_trans0213", false}, {"config.is_interleaved", false}, {"config.rotary_ndims", rotary_ndims}, {"config.is_chatglm", true}, @@ -646,6 +654,7 @@ TEST_F(TransformationTestsF, ConvertToROPE_chatGML_Slice) { {{"config.slice_start", 0}, {"config.slice_stop", 4096}, {"config.input_trans0213", false}, + {"config.output_trans0213", false}, {"config.is_interleaved", false}, {"config.rotary_ndims", rotary_ndims}, {"config.is_chatglm", true}, @@ -728,6 +737,7 @@ TEST_F(TransformationTestsF, ConvertToROPE_GPTJ_Slice) { {{"config.slice_start", 0}, {"config.slice_stop", 0}, {"config.input_trans0213", false}, + {"config.output_trans0213", true}, {"config.is_interleaved", true}, {"config.is_chatglm", false}, {"config.support_2d_rope", false}, @@ -843,6 +853,7 @@ TEST_F(TransformationTestsF, ConvertToROPE_chatGML_2d_rope) { {{"config.slice_start", 0}, {"config.slice_stop", 4096}, {"config.input_trans0213", false}, + {"config.output_trans0213", false}, {"config.is_interleaved", false}, {"config.rotary_ndims", rotary_ndims}, {"config.is_chatglm", true}, @@ -951,6 +962,7 @@ TEST_F(TransformationTestsF, ConvertToROPE_chatGML_nano_2d_rope) { {{"config.slice_start", 0}, {"config.slice_stop", 2048}, {"config.input_trans0213", false}, + {"config.output_trans0213", false}, {"config.is_interleaved", false}, {"config.rotary_ndims", rotary_ndims}, {"config.is_chatglm", true}, @@ -962,4 +974,160 @@ TEST_F(TransformationTestsF, ConvertToROPE_chatGML_nano_2d_rope) { model_ref = std::make_shared(ov::NodeVector{rope}, ov::ParameterVector{input, cos_sin_cache, position_ids}); } -} \ No newline at end of file +} + +TEST_F(TransformationTestsF, ConvertToROPE_Flux_mul) { + disable_rt_info_check(); + const int batch = 2; + const int num_heads = 32; + const int ndims = 128; + { + auto x = + std::make_shared(ov::element::f32, ov::PartialShape{batch, num_heads, -1, ndims}); + auto t_cos = std::make_shared(ov::element::f32, ov::PartialShape{1, 1, -1, ndims}); + auto t_sin = std::make_shared(ov::element::f32, ov::PartialShape{1, 1, -1, ndims}); + + auto x1_shape = makeConst(ov::element::i64, ov::Shape({5}), {0, num_heads, 0, -1, 2}); + auto x1 = std::make_shared(x, x1_shape, true); + + auto split_axis = makeConst(ov::element::i64, ov::Shape(), {-1}); + auto split = std::make_shared(x1, split_axis, 2); + + auto minus_one = makeConst(ov::element::f32, ov::Shape({}), {-1.0f}); + auto x1_1_neg = std::make_shared(split->output(1), minus_one); + + auto x2 = std::make_shared(ov::OutputVector{x1_1_neg->output(0), split->output(0)}, -1); + + auto x3_shape = makeConst(ov::element::i64, ov::Shape({4}), {0, num_heads, 0, ndims}); + auto x3 = std::make_shared(x2, x3_shape, true); + + auto y1 = std::make_shared(x, t_cos); + auto y2 = std::make_shared(x3, t_sin); + auto y = std::make_shared(y1, y2); + + model = std::make_shared(ov::NodeVector{y}, ov::ParameterVector{x, t_cos, t_sin}); + } + manager.register_pass(true); + { + auto x = + std::make_shared(ov::element::f32, ov::PartialShape{batch, num_heads, -1, ndims}); + auto t_cos = std::make_shared(ov::element::f32, ov::PartialShape{1, 1, -1, ndims}); + auto t_sin = std::make_shared(ov::element::f32, ov::PartialShape{1, 1, -1, ndims}); + ov::op::internal::RoPE::Config config; + config.is_interleaved = true; + config.rotary_ndims = ndims; + config.head_cnt = num_heads; + config.head_size = ndims; + auto rope = std::make_shared(ov::OutputVector{x, t_cos, t_sin}, config); + model_ref = std::make_shared(ov::NodeVector{rope}, ov::ParameterVector{x, t_cos, t_sin}); + } + comparator.enable(FunctionsComparator::ATTRIBUTES); +} + +TEST_F(TransformationTestsF, ConvertToROPE_Flux_squeeze_mul_unsqueeze) { + disable_rt_info_check(); + const int batch = 2; + const int num_heads = 32; + const int ndims = 128; + { + auto x = + std::make_shared(ov::element::f32, ov::PartialShape{batch, num_heads, -1, ndims}); + auto t_cos = std::make_shared(ov::element::f32, ov::PartialShape{1, 1, -1, ndims}); + auto t_sin = std::make_shared(ov::element::f32, ov::PartialShape{1, 1, -1, ndims}); + + auto x1_shape = makeConst(ov::element::i64, ov::Shape({5}), {0, num_heads, 0, -1, 2}); + auto x1 = std::make_shared(x, x1_shape, true); + + auto split_axis = makeConst(ov::element::i64, ov::Shape(), {-1}); + auto split = std::make_shared(x1, split_axis, 2); + + auto squeeze_axis = makeConst(ov::element::i32, ov::Shape({}), {-1}); + auto squeeze = std::make_shared(split->output(1), squeeze_axis); + + auto minus_one = makeConst(ov::element::f32, ov::Shape({}), {-1.0f}); + auto x1_1_neg = std::make_shared(squeeze, minus_one); + + auto unsqueeze_axis = makeConst(ov::element::i32, ov::Shape({}), {-1}); + auto unsqueeze = std::make_shared(x1_1_neg, unsqueeze_axis); + + auto x2 = std::make_shared(ov::OutputVector{unsqueeze->output(0), split->output(0)}, -1); + + auto x3_shape = makeConst(ov::element::i64, ov::Shape({4}), {0, num_heads, 0, ndims}); + auto x3 = std::make_shared(x2, x3_shape, true); + + auto y1 = std::make_shared(x, t_cos); + auto y2 = std::make_shared(x3, t_sin); + auto y = std::make_shared(y1, y2); + + model = std::make_shared(ov::NodeVector{y}, ov::ParameterVector{x, t_cos, t_sin}); + } + manager.register_pass(true); + { + auto x = + std::make_shared(ov::element::f32, ov::PartialShape{batch, num_heads, -1, ndims}); + auto t_cos = std::make_shared(ov::element::f32, ov::PartialShape{1, 1, -1, ndims}); + auto t_sin = std::make_shared(ov::element::f32, ov::PartialShape{1, 1, -1, ndims}); + ov::op::internal::RoPE::Config config; + config.is_interleaved = true; + config.rotary_ndims = ndims; + config.head_cnt = num_heads; + config.head_size = ndims; + auto rope = std::make_shared(ov::OutputVector{x, t_cos, t_sin}, config); + model_ref = std::make_shared(ov::NodeVector{rope}, ov::ParameterVector{x, t_cos, t_sin}); + } + comparator.enable(FunctionsComparator::ATTRIBUTES); +} + +TEST_F(TransformationTestsF, ConvertToROPE_Flux_mul_squeeze_unsqueeze) { + disable_rt_info_check(); + const int batch = 2; + const int num_heads = 32; + const int ndims = 128; + { + auto x = + std::make_shared(ov::element::f32, ov::PartialShape{batch, num_heads, -1, ndims}); + auto t_cos = std::make_shared(ov::element::f32, ov::PartialShape{1, 1, -1, ndims}); + auto t_sin = std::make_shared(ov::element::f32, ov::PartialShape{1, 1, -1, ndims}); + + auto x1_shape = makeConst(ov::element::i64, ov::Shape({5}), {0, num_heads, 0, -1, 2}); + auto x1 = std::make_shared(x, x1_shape, true); + + auto split_axis = makeConst(ov::element::i64, ov::Shape(), {-1}); + auto split = std::make_shared(x1, split_axis, 2); + + auto minus_one = makeConst(ov::element::f32, ov::Shape({}), {-1.0f}); + auto x1_1_neg = std::make_shared(split->output(1), minus_one); + + auto squeeze_axis = makeConst(ov::element::i32, ov::Shape({}), {-1}); + auto squeeze = std::make_shared(x1_1_neg, squeeze_axis); + + auto unsqueeze_axis = makeConst(ov::element::i32, ov::Shape({}), {-1}); + auto unsqueeze = std::make_shared(squeeze, unsqueeze_axis); + + auto x2 = std::make_shared(ov::OutputVector{unsqueeze->output(0), split->output(0)}, -1); + + auto x3_shape = makeConst(ov::element::i64, ov::Shape({4}), {0, num_heads, 0, ndims}); + auto x3 = std::make_shared(x2, x3_shape, true); + + auto y1 = std::make_shared(x, t_cos); + auto y2 = std::make_shared(x3, t_sin); + auto y = std::make_shared(y1, y2); + + model = std::make_shared(ov::NodeVector{y}, ov::ParameterVector{x, t_cos, t_sin}); + } + manager.register_pass(true); + { + auto x = + std::make_shared(ov::element::f32, ov::PartialShape{batch, num_heads, -1, ndims}); + auto t_cos = std::make_shared(ov::element::f32, ov::PartialShape{1, 1, -1, ndims}); + auto t_sin = std::make_shared(ov::element::f32, ov::PartialShape{1, 1, -1, ndims}); + ov::op::internal::RoPE::Config config; + config.is_interleaved = true; + config.rotary_ndims = ndims; + config.head_cnt = num_heads; + config.head_size = ndims; + auto rope = std::make_shared(ov::OutputVector{x, t_cos, t_sin}, config); + model_ref = std::make_shared(ov::NodeVector{rope}, ov::ParameterVector{x, t_cos, t_sin}); + } + comparator.enable(FunctionsComparator::ATTRIBUTES); +} diff --git a/src/common/transformations/tests/common_optimizations/remove_multi_subgraph_op_dangling_params_tests.cpp b/src/common/transformations/tests/common_optimizations/remove_multi_subgraph_op_dangling_params_tests.cpp index 89f332bffebff8..4e8ad1765bd20f 100644 --- a/src/common/transformations/tests/common_optimizations/remove_multi_subgraph_op_dangling_params_tests.cpp +++ b/src/common/transformations/tests/common_optimizations/remove_multi_subgraph_op_dangling_params_tests.cpp @@ -175,7 +175,7 @@ TEST_F(TransformationTestsF, RemoveLoopDanglingParametersIfConcatEmptyTensor) { } } -TEST_F(TransformationTestsF, RemoveIfDanglingParametersFromBodiesAndInputs) { +TEST_F(TransformationTestsF, RemoveIfDanglingParametersFromBodiesAndInputsConsecutive) { auto X = std::make_shared(element::f32, Shape{2, 4, 1}); auto Y = std::make_shared(element::f32, Shape{3, 4, 1}); auto cond = std::make_shared(element::boolean, Shape{1}, true); @@ -196,6 +196,8 @@ TEST_F(TransformationTestsF, RemoveIfDanglingParametersFromBodiesAndInputs) { if_op->set_else_body(else_body); if_op->set_input(X, Xte, Xte); if_op->set_input(Y, Yte, Yte); + // if_op descriptors are [desc_0, desc_1, desc_2, desc_3] + // desc_0, desc_2 are dangling, Parameters Y, Yte should be removed auto res = if_op->set_output(then_op_res, else_op_res); model = std::make_shared(OutputVector{res}, ParameterVector{X, Y}); @@ -213,6 +215,46 @@ TEST_F(TransformationTestsF, RemoveIfDanglingParametersFromBodiesAndInputs) { } } +TEST_F(TransformationTestsF, RemoveIfDanglingParametersFromBodiesAndInputsNotConsecutive) { + auto X = std::make_shared(element::f32, Shape{2, 4, 1}); + auto Y = std::make_shared(element::f32, Shape{3, 4, 1}); + auto cond = std::make_shared(element::boolean, Shape{1}, false); + + auto Xte = std::make_shared(element::f32, PartialShape::dynamic()); + auto Yte = std::make_shared(element::f32, PartialShape::dynamic()); + + auto then_op = std::make_shared(Yte, Yte); + auto then_op_res = std::make_shared(then_op); + + auto else_op = std::make_shared(Yte, Yte); + auto else_op_res = std::make_shared(else_op); + { + auto then_body = std::make_shared(OutputVector{then_op_res}, ParameterVector{Xte, Yte}); + auto else_body = std::make_shared(OutputVector{else_op_res}, ParameterVector{Xte, Yte}); + auto if_op = std::make_shared(cond); + if_op->set_then_body(then_body); + if_op->set_else_body(else_body); + if_op->set_input(X, Xte, Yte); + if_op->set_input(Y, Xte, Xte); + // if_op descriptors are [desc_0, desc_1, desc_2, desc_3] + // desc_0, desc_2, desc_3 are dangling, Parameters Y, Xte should be removed + auto res = if_op->set_output(then_op_res, else_op_res); + model = std::make_shared(OutputVector{res}, ParameterVector{X, Y}); + + manager.register_pass(); + } + { + auto then_body = std::make_shared(OutputVector{then_op_res}, ParameterVector{Yte}); + auto else_body = std::make_shared(OutputVector{else_op_res}, ParameterVector{Yte}); + auto if_op = std::make_shared(cond); + if_op->set_then_body(then_body); + if_op->set_else_body(else_body); + if_op->set_input(X, Yte, Yte); + auto res = if_op->set_output(then_op_res, else_op_res); + model_ref = std::make_shared(OutputVector{res}, ParameterVector{X, Y}); + } +} + TEST_F(TransformationTestsF, RemoveIfDanglingParametersOnlyFromBodies) { auto X = std::make_shared(element::f32, Shape{2, 4, 1}); auto Y = std::make_shared(element::f32, Shape{3, 4, 1}); @@ -518,23 +560,28 @@ TEST_F(TransformationTestsF, RemoveLoopDanglingParamsAndResults) { auto ai = std::make_shared(element::f32, Shape{2, 2}); auto b = std::make_shared(element::f32, Shape{2, 2}); auto bi = std::make_shared(element::f32, Shape{2, 2}); + auto c = std::make_shared(element::f32, Shape{2, 2}); + auto ci = std::make_shared(element::f32, Shape{2, 2}); + auto d = std::make_shared(element::f32, Shape{2, 2}); auto mul = std::make_shared(ai, ai); auto abs1 = std::make_shared(mul); auto add = std::make_shared(bi, bi); auto abs2 = std::make_shared(add); { - auto body = std::make_shared(OutputVector{condition, abs1, abs2}, ParameterVector{ai, bi}); + auto body = std::make_shared(OutputVector{condition, abs1, abs2}, ParameterVector{ai, bi, ci}); auto loop = std::make_shared(trip_count, condition); loop->set_special_body_ports({-1, 0}); loop->set_function(body); loop->set_invariant_input(ai, a); + loop->set_invariant_input(ci, d); loop->set_invariant_input(bi, b); + loop->set_invariant_input(ci, c); auto loop_res = std::make_shared(loop->get_iter_value(abs1)); loop->get_iter_value(abs2); // abs2 result is unused - model = std::make_shared(OutputVector{loop_res}, ParameterVector{a, b}); + model = std::make_shared(OutputVector{loop_res}, ParameterVector{a, b, c, d}); manager.register_pass(); } diff --git a/src/common/util/include/openvino/util/common_util.hpp b/src/common/util/include/openvino/util/common_util.hpp index d068db373360b3..312aab17419af4 100644 --- a/src/common/util/include/openvino/util/common_util.hpp +++ b/src/common/util/include/openvino/util/common_util.hpp @@ -15,24 +15,42 @@ namespace ov { namespace util { -template -std::string join(const T& v, const std::string& sep = ", ") { +/** + * @brief Join container's elements to string using user string as separator. + * + * @param container Element to make joined string. + * @param sep User string used as separator. Default ", ". + * @return Joined elements as string. + */ +template +std::string join(const Container& container, const std::string& sep = ", ") { std::ostringstream ss; - size_t count = 0; - for (const auto& x : v) { - if (count++ > 0) { - ss << sep; + auto first = std::begin(container); + const auto last = std::end(container); + if (first != last) { + ss << *first; + ++first; + for (; first != last; ++first) { + ss << sep << *first; } - ss << x; } return ss.str(); } -template -std::string vector_to_string(const T& v) { - std::ostringstream os; - os << "[ " << ov::util::join(v) << " ]"; - return os.str(); +/** + * @brief Stringify the input vector. + * + * The vector is converted to the string as "[ element 0, element 1, ..., element N ]". + * Examples: + * - std::vector{1,3,5} -> "[ 1, 3, 5 ]" + * - std::vector{} -> "[ ]" + * + * @param v Vector to be converted + * @return String contains + */ +template +std::string vector_to_string(const std::vector& v) { + return "[ " + ov::util::join(v) + " ]"; } std::string to_lower(const std::string& s); @@ -113,11 +131,16 @@ T ceil_div(const T& x, const T& y) { return (x == 0 ? 0 : (1 + (x - 1) / y)); } -template -bool contains(const std::vector& vec, const V& v) { - return std::any_of(vec.begin(), vec.end(), [&](const T& x) { - return x == v; - }); +/** + * @brief Checks if container contains the specific value. + * + * @param container The container of elements to examine. + * @param value Value to compare the elements to. + * @return True if value found in the container, false otherwise. + */ +template +bool contains(const R& container, const V& value) { + return std::find(std::begin(container), std::end(container), value) != std::end(container); } /** @@ -127,12 +150,7 @@ bool contains(const std::vector& vec, const V& v) { */ template T product(std::vector const& vec) { - if (vec.empty()) - return 0; - T ret = vec[0]; - for (size_t i = 1; i < vec.size(); ++i) - ret *= vec[i]; - return ret; + return vec.empty() ? T{0} : std::accumulate(vec.begin(), vec.end(), T{1}, std::multiplies()); } /** diff --git a/src/core/include/openvino/core/descriptor/tensor.hpp b/src/core/include/openvino/core/descriptor/tensor.hpp index a1cac56fa64e01..9624994d8d612e 100644 --- a/src/core/include/openvino/core/descriptor/tensor.hpp +++ b/src/core/include/openvino/core/descriptor/tensor.hpp @@ -107,7 +107,6 @@ class OPENVINO_API Tensor { friend OPENVINO_API void set_tensor_type(Tensor& tensor, const element::Type& element_type, const PartialShape& pshape); - friend class pass::ReverseShapeAndTypeInfer; private: mutable std::atomic m_shape_changing{false}; diff --git a/src/core/reference/include/openvino/reference/reduce_mean.hpp b/src/core/reference/include/openvino/reference/reduce_mean.hpp index 4c46d4ca786d09..f046f4f96197bb 100644 --- a/src/core/reference/include/openvino/reference/reduce_mean.hpp +++ b/src/core/reference/include/openvino/reference/reduce_mean.hpp @@ -26,6 +26,10 @@ void reduce_mean(const T* in, T* out, const Shape& in_shape, const AxisSet& redu reduce_sum(in, out, in_shape, reduction_axes); const auto out_shape = util::reduce(in_shape, reduction_axes); + if (shape_size(in_shape) == 0) { + return; + } + const auto out_size = shape_size(out_shape); const auto count = static_cast(shape_size(in_shape) / out_size); std::transform(out, std::next(out, out_size), out, [count](const T value) { diff --git a/src/core/reference/include/openvino/reference/search_sorted.hpp b/src/core/reference/include/openvino/reference/search_sorted.hpp index 7ea8ec1078a2a1..629509b28ef78d 100644 --- a/src/core/reference/include/openvino/reference/search_sorted.hpp +++ b/src/core/reference/include/openvino/reference/search_sorted.hpp @@ -32,6 +32,7 @@ void search_sorted(const T* sorted, } const size_t size = shape_size(values_shape); + const size_t sorted_inner_dim = sorted_shape.back(); auto func = [&](size_t i) { auto it = values_transform.begin(); @@ -44,15 +45,12 @@ void search_sorted(const T* sorted, Coordinate sorted_coord_begin = values_coord; sorted_coord_begin.back() = 0; - Coordinate sorted_coord_last = values_coord; - sorted_coord_last.back() = sorted_shape.back(); - const auto sorted_index_begin = coordinate_index(sorted_coord_begin, sorted_shape); - const auto sorted_index_last = coordinate_index(sorted_coord_last, sorted_shape); - - const T* idx_ptr = compare_func(sorted + sorted_index_begin, sorted + sorted_index_last, value); + const T* sorted_begin_ptr = sorted + sorted_index_begin; + const T* sorted_end_ptr = sorted_begin_ptr + sorted_inner_dim; + const T* idx_ptr = compare_func(sorted_begin_ptr, sorted_end_ptr, value); - const ptrdiff_t sorted_index = (idx_ptr - sorted) - sorted_index_begin; + const ptrdiff_t sorted_index = idx_ptr - sorted_begin_ptr; out[values_index] = static_cast(sorted_index); }; diff --git a/src/frontends/onnx/frontend/src/op/com.microsoft/fusedmatmul.cpp b/src/frontends/onnx/frontend/src/op/com.microsoft/fusedmatmul.cpp new file mode 100644 index 00000000000000..d141b47c23213c --- /dev/null +++ b/src/frontends/onnx/frontend/src/op/com.microsoft/fusedmatmul.cpp @@ -0,0 +1,85 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#include "cmath" +#include "core/operator_set.hpp" +#include "exceptions.hpp" +#include "openvino/frontend/exception.hpp" +#include "openvino/op/constant.hpp" +#include "openvino/op/matmul.hpp" +#include "openvino/op/multiply.hpp" +#include "openvino/op/transpose.hpp" +#include "utils/common.hpp" + +using namespace ov::op; + +namespace ov { +namespace frontend { +namespace onnx { +namespace com_microsoft { +namespace opset_1 { + +std::vector get_transpose_axes(size_t rank) { + std::vector axes(rank); + std::iota(axes.begin(), axes.end(), 0); + + if (rank > 2) { + std::rotate(axes.begin() + 1, axes.begin() + 2, axes.begin() + rank - 1); + } + return axes; +} + +ov::OutputVector fusedmatmul(const ov::frontend::onnx::Node& node) { + common::default_op_checks(node, 2); + + const auto inputs = node.get_ov_inputs(); + auto A = inputs[0]; // required + auto B = inputs[1]; // required + + const auto alpha = node.get_attribute_value("alpha"); // required + const auto transA = node.get_attribute_value("transA"); // required + const auto transB = node.get_attribute_value("transB"); // required + const auto transBatchA = node.get_attribute_value("transBatchA"); // required + const auto transBatchB = node.get_attribute_value("transBatchB"); // required + + CHECK_VALID_NODE(node, + A.get_element_type() == ov::element::f16 || A.get_element_type() == ov::element::f32 || + A.get_element_type() == ov::element::f64 || A.get_element_type() == ov::element::bf16, + "Unsupported input A type, accepted FP16, FP32, FP64, BFP16 got: ", + A.get_element_type()); + CHECK_VALID_NODE(node, + B.get_element_type() == ov::element::f16 || B.get_element_type() == ov::element::f32 || + B.get_element_type() == ov::element::f64 || B.get_element_type() == ov::element::bf16, + "Unsupported input B type, accepted FP16, FP32, FP64, BFP16 got: ", + B.get_element_type()); + + const auto rankA = A.get_shape().size(); + const auto rankB = B.get_shape().size(); + + if (transBatchA) { + A = std::make_shared( + A, + std::make_shared(element::i64, Shape{rankA}, get_transpose_axes(rankA))); + } + + if (transBatchB) { + B = std::make_shared( + B, + std::make_shared(element::i64, Shape{rankB}, get_transpose_axes(rankB))); + } + + auto matmul_result = std::make_shared(A, B, transA, transB); + + auto alpha_const = std::make_shared(A.get_element_type(), Shape{}, std::vector{alpha}); + auto scaled_result = std::make_shared(matmul_result, alpha_const); + + return {scaled_result}; +} + +ONNX_OP("FusedMatMul", OPSET_SINCE(1), com_microsoft::opset_1::fusedmatmul, MICROSOFT_DOMAIN); + +} // namespace opset_1 +} // namespace com_microsoft +} // namespace onnx +} // namespace frontend +} // namespace ov diff --git a/src/frontends/onnx/frontend/src/op/com.microsoft/skip_simplified_layer_normalization.cpp b/src/frontends/onnx/frontend/src/op/com.microsoft/skip_simplified_layer_normalization.cpp new file mode 100644 index 00000000000000..9872d323219271 --- /dev/null +++ b/src/frontends/onnx/frontend/src/op/com.microsoft/skip_simplified_layer_normalization.cpp @@ -0,0 +1,79 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "core/operator_set.hpp" +#include "exceptions.hpp" +#include "openvino/frontend/exception.hpp" +#include "openvino/op/add.hpp" +#include "openvino/op/convert.hpp" +#include "openvino/op/divide.hpp" +#include "openvino/op/multiply.hpp" +#include "openvino/op/power.hpp" +#include "openvino/op/range.hpp" +#include "openvino/op/reduce_mean.hpp" +#include "openvino/op/reduce_sum.hpp" +#include "openvino/op/shape_of.hpp" +#include "openvino/op/sqrt.hpp" +#include "utils/common.hpp" + +using namespace ov::op; +using ::ONNX_NAMESPACE::TensorProto_DataType; + +namespace ov { +namespace frontend { +namespace onnx { +namespace com_microsoft { +namespace opset_1 { + +ov::OutputVector skip_simplified_layer_normalization(const ov::frontend::onnx::Node& node) { + common::default_op_checks(node, 3); + + const auto inputs = node.get_ov_inputs(); + const auto input_tensor = inputs[0]; + const auto skip = inputs[1]; + + CHECK_VALID_NODE(node, + input_tensor.get_element_type() == skip.get_element_type(), + "input tensor and skip must be of same type, got :", + input_tensor.get_element_type(), + skip.get_element_type()); + + // input + skip + std::shared_ptr input = std::make_shared(input_tensor, skip); + + // add bias if available + if (inputs.size() == 4) { + const auto bias = inputs[3]; + CHECK_VALID_NODE(node, + input_tensor.get_element_type() == bias.get_element_type(), + "input tensor and bias must be of same type, got: ", + input_tensor.get_element_type(), + bias.get_element_type()); + input = std::make_shared(input, bias); + } + + float epsilon = node.get_attribute_value("epsilon"); + ov::element::Type element_type = input->get_output_element_type(0); + + auto squared_input = std::make_shared(input, input); + auto mean = std::make_shared(squared_input, + v0::Constant::create(element::i64, {}, {-1}), + true); // mean = (1/N) * Σ(j=1 to N) X_j^2 + auto rms_value = + std::make_shared(std::make_shared(mean, v0::Constant::create(element_type, {}, {epsilon}))); + auto inv_std_var = std::make_shared(v0::Constant::create(element_type, {}, {1.0f}), rms_value); + auto normalized = std::make_shared(input, inv_std_var); // X / RMS(X) auto scaled = + auto scaled = std::make_shared(normalized, inputs[2]); // (X / RMS(X)) * scale + + return ov::OutputVector{scaled, mean, inv_std_var, input}; +} +ONNX_OP("SkipSimplifiedLayerNormalization", + OPSET_SINCE(1), + com_microsoft::opset_1::skip_simplified_layer_normalization, + MICROSOFT_DOMAIN); +} // namespace opset_1 +} // namespace com_microsoft +} // namespace onnx +} // namespace frontend +} // namespace ov diff --git a/src/frontends/onnx/tests/models/com.microsoft/fusedmatmul_2D.prototxt b/src/frontends/onnx/tests/models/com.microsoft/fusedmatmul_2D.prototxt new file mode 100644 index 00000000000000..9d7a739f25b6c7 --- /dev/null +++ b/src/frontends/onnx/tests/models/com.microsoft/fusedmatmul_2D.prototxt @@ -0,0 +1,93 @@ +ir_version: 3 +producer_name: "OpenVINO ONNX Frontend" +model_version: 0 +graph { + name: "FusedMatMulModel" + node { + op_type: "FusedMatMul" + input: "A" + input: "B" + output: "output" + attribute { + name: "alpha" + f: 0.75 + type: FLOAT + } + attribute { + name: "transA" + i: 1 + type: INT + } + attribute { + name: "transB" + i: 0 + type: INT + } + attribute { + name: "transBatchA" + i: 0 + type: INT + } + attribute { + name: "transBatchB" + i: 0 + type: INT + } + domain: "com.microsoft" + } + input { + name: "A" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 3 + } + dim { + dim_value: 2 + } + } + } + } + } + input { + name: "B" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 3 + } + dim { + dim_value: 2 + } + } + } + } + } + output { + name: "output" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 2 + } + dim { + dim_value: 2 + } + } + } + } + } +} +opset_import { + version: 7 +} +opset_import { + version: 1 + domain: "com.microsoft" +} diff --git a/src/frontends/onnx/tests/models/com.microsoft/fusedmatmul_trans_and_transbatch_enabled.prototxt b/src/frontends/onnx/tests/models/com.microsoft/fusedmatmul_trans_and_transbatch_enabled.prototxt new file mode 100644 index 00000000000000..07c19d71fbb550 --- /dev/null +++ b/src/frontends/onnx/tests/models/com.microsoft/fusedmatmul_trans_and_transbatch_enabled.prototxt @@ -0,0 +1,102 @@ +ir_version: 3 +producer_name: "OpenVINO ONNX Frontend" +model_version: 0 +graph { + name: "FusedMatMulModel" + node { + op_type: "FusedMatMul" + input: "A" + input: "B" + output: "output" + attribute { + name: "alpha" + f: 0.5 + type: FLOAT + } + attribute { + name: "transA" + i: 1 + type: INT + } + attribute { + name: "transB" + i: 1 + type: INT + } + attribute { + name: "transBatchA" + i: 1 + type: INT + } + attribute { + name: "transBatchB" + i: 1 + type: INT + } + domain: "com.microsoft" + } + input { + name: "A" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 2 + } + dim { + dim_value: 5 + } + dim { + dim_value: 4 + } + } + } + } + } + input { + name: "B" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 2 + } + dim { + dim_value: 3 + } + dim { + dim_value: 5 + } + } + } + } + } + output { + name: "output" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 2 + } + dim { + dim_value: 4 + } + dim { + dim_value: 3 + } + } + } + } + } +} +opset_import { + version: 7 +} +opset_import { + version: 1 + domain: "com.microsoft" +} diff --git a/src/frontends/onnx/tests/models/com.microsoft/skip_simplified_layer_normalization.prototxt b/src/frontends/onnx/tests/models/com.microsoft/skip_simplified_layer_normalization.prototxt new file mode 100644 index 00000000000000..fbaaae6e9cc5a4 --- /dev/null +++ b/src/frontends/onnx/tests/models/com.microsoft/skip_simplified_layer_normalization.prototxt @@ -0,0 +1,80 @@ +ir_version: 6 +producer_name: "OpenVINO ONNX Frontend" +graph { + node { + input: "input" + input: "skip" + input: "gamma" + input: "bias" + output: "out" + name: "SkipSimplifiedLayerNorm" + op_type: "SkipSimplifiedLayerNormalization" + attribute { + name: "epsilon" + f: 1e-05 + type: FLOAT + } + domain: "com.microsoft" + } + initializer { + dims: 4 + data_type: 1 # FLOAT32 + name: "gamma" + float_data: 0.1 + float_data: 0.2 + float_data: 0.3 + float_data: 0.4 + } + initializer { + dims: 4 + data_type: 1 # FLOAT32 + name: "bias" + float_data: -0.07 + float_data: -0.4 + float_data: 0.22 + float_data: 0.0 + } + input { + name: "input" + type { + tensor_type { + elem_type: 1 # FLOAT32 + shape { + dim { dim_value: 3 } + dim { dim_value: 2 } + dim { dim_value: 4 } + } + } + } + } + input { + name: "skip" + type { + tensor_type { + elem_type: 1 # FLOAT32 + shape { + dim { dim_value: 3 } + dim { dim_value: 2 } + dim { dim_value: 4 } + } + } + } + } + output { + name: "out" + type { + tensor_type { + elem_type: 1 # FLOAT32 + shape { + dim { dim_value: 3 } + dim { dim_value: 2 } + dim { dim_value: 4 } + } + } + } + } +} +opset_import { + domain: "com.microsoft" + version: 1 +} diff --git a/src/frontends/onnx/tests/onnx_import_com_microsoft.in.cpp b/src/frontends/onnx/tests/onnx_import_com_microsoft.in.cpp index cdaa7c392c6798..ca805f4c194856 100644 --- a/src/frontends/onnx/tests/onnx_import_com_microsoft.in.cpp +++ b/src/frontends/onnx/tests/onnx_import_com_microsoft.in.cpp @@ -1357,6 +1357,28 @@ OPENVINO_TEST(${BACKEND_NAME}, onnx_com_microsoft_quickgelu) { } } +OPENVINO_TEST(${BACKEND_NAME}, onnx_model_skip_simplified_layer_normalization) { + const auto model = convert_model("com.microsoft/skip_simplified_layer_normalization.onnx"); + auto test_case = ov::test::TestCase(model, s_device); + + std::vector input = {1.f, 2.f, 3.f, 4.f, 5.f, 6.f, 7.f, 8.f, 9.f, 10.f, 11.f, 12.f, + 13.f, 14.f, 15.f, 16.f, 17.f, 18.f, 19.f, 20.f, 21.f, 22.f, 23.f, 24.f}; + std::vector skip = {1.f, 2.f, 3.f, 4.f, 5.f, 6.f, 7.f, 8.f, 9.f, 10.f, 11.f, 12.f, + 13.f, 14.f, 15.f, 16.f, 17.f, 18.f, 19.f, 20.f, 21.f, 22.f, 23.f, 24.f}; + std::vector expected_output = { + 0.0353291891515255f, 0.1317980140447617f, 0.3415765166282654f, 0.5857689380645752f, 0.07553060352802277f, + 0.1764662563800812f, 0.3244849443435669f, 0.4868034422397614f, 0.08510704338550568f, 0.1860678195953369f, + 0.3164101839065552f, 0.4556762874126434f, 0.08931596577167511f, 0.1901365667581558f, 0.3122786283493042f, + 0.4408963620662689f, 0.09167447686195374f, 0.19237320125103f, 0.3097965121269226f, 0.4322993457317352f, + 0.09318139404058456f, 0.1937852799892426f, 0.3081453144550323f, 0.4266831874847412f}; + + test_case.add_input(Shape{3, 2, 4}, input); + test_case.add_input(Shape{3, 2, 4}, skip); + test_case.add_expected_output({3, 2, 4}, expected_output); + + test_case.run(); +} + OPENVINO_TEST(${BACKEND_NAME}, onnx_com_microsoft_simplified_layer_normalization_2x2x8) { const auto model = convert_model("com.microsoft/simplified_layer_normalization_2x2x8.onnx"); auto test_case = ov::test::TestCase(model, s_device); @@ -1395,3 +1417,42 @@ OPENVINO_TEST(${BACKEND_NAME}, onnx_com_microsoft_simplified_layer_normalization test_case.run(); } + +OPENVINO_TEST(${BACKEND_NAME}, onnx_com_microsoft_fusedmatmul_2x3) { + const auto model = convert_model("com.microsoft/fusedmatmul_2D.onnx"); + auto test_case = ov::test::TestCase(model, s_device); + + const std::vector data_A{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; + const std::vector data_B{7.0f, 8.0f, 9.0f, 10.0f, 11.0f, 12.0f}; + const std::vector expected_output{66.75f, 73.5f, 87.f, 96.f}; + + test_case.add_input(Shape{3, 2}, data_A); + test_case.add_input(Shape{3, 2}, data_B); + test_case.add_expected_output(Shape{2, 2}, expected_output); + + test_case.run(); +} + +OPENVINO_TEST(${BACKEND_NAME}, onnx_com_microsoft_fusedmatmul_2x5x3x6x4_2x6x3x4x7) { + const auto model = convert_model("com.microsoft/fusedmatmul_trans_and_transbatch_enabled.onnx"); + auto test_case = ov::test::TestCase(model, s_device); + + const std::vector data_A = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f, + 11.0f, 12.0f, 13.0f, 14.0f, 15.0f, 16.0f, 17.0f, 18.0f, 19.0f, 20.0f, + 21.0f, 22.0f, 23.0f, 24.0f, 25.0f, 26.0f, 27.0f, 28.0f, 29.0f, 30.0f, + 31.0f, 32.0f, 33.0f, 34.0f, 35.0f, 36.0f, 37.0f, 38.0f, 39.0f, 40.0f}; + + const std::vector data_B = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f, + 11.0f, 12.0f, 13.0f, 14.0f, 15.0f, 16.0f, 17.0f, 18.0f, 19.0f, 20.0f, + 21.0f, 22.0f, 23.0f, 24.0f, 25.0f, 26.0f, 27.0f, 28.0f, 29.0f, 30.0f}; + + const std::vector expected_output = {87.5f, 200.0f, 312.5f, 95.0f, 220.0f, 345.0f, 102.5f, 240.0f, + 377.5f, 110.0f, 260.0f, 410.0f, 1325.0f, 1687.5f, 2050.0f, 1370.0f, + 1745.0f, 2120.0f, 1415.0f, 1802.5f, 2190.0f, 1460.0f, 1860.0f, 2260.0f}; + + test_case.add_input(Shape{2, 5, 4}, data_A); + test_case.add_input(Shape{2, 3, 5}, data_B); + test_case.add_expected_output(Shape{2, 4, 3}, expected_output); + + test_case.run(); +} diff --git a/src/frontends/pytorch/src/op/index_fill_.cpp b/src/frontends/pytorch/src/op/index_fill_.cpp new file mode 100644 index 00000000000000..a24f3fa2f5b1c7 --- /dev/null +++ b/src/frontends/pytorch/src/op/index_fill_.cpp @@ -0,0 +1,69 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include "openvino/core/node.hpp" +#include "openvino/core/node_output.hpp" +#include "openvino/core/type/element_type.hpp" +#include "openvino/frontend/pytorch/node_context.hpp" +#include "openvino/op/broadcast.hpp" +#include "openvino/op/constant.hpp" +#include "openvino/op/convert_like.hpp" +#include "openvino/op/reshape.hpp" +#include "openvino/op/scatter_elements_update.hpp" +#include "openvino/op/shape_of.hpp" +#include "openvino/op/slice.hpp" +#include "utils.hpp" + +using namespace ov::op; + +namespace ov { +namespace frontend { +namespace pytorch { +namespace op { + +OutputVector translate_index_fill_(const NodeContext& context) { + // aten::index_fill_(self, dim, index, value) --> Tensor + num_inputs_check(context, 4, 4); + auto input = context.get_input(0); + auto dim = context.get_input(1); + auto index = context.get_input(2); + auto value = context.get_input(3); + + auto const_1_vec = v0::Constant::create(element::i32, Shape{1}, {1}); + + auto tensor_rank = std::get<1>(get_shape_rank(context, input, false)); + auto tensor_rank_correct_type = context.mark_node(std::make_shared(tensor_rank, dim)); + auto dim_vec = normalize_axis(context, dim, tensor_rank_correct_type); + + // scalar to vec + auto value_vec = context.mark_node(std::make_shared(value, const_1_vec, false)); + + auto input_shape = std::get<0>(get_shape_rank(context, input, false)); + + auto index_shape = std::get<0>(get_shape_rank(context, index, false)); + auto const_0 = context.mark_node(v0::Constant::create(element::i32, Shape{1}, {0})); + auto const_1 = context.mark_node(v0::Constant::create(element::i32, Shape{1}, {1})); + auto index_len = context.mark_node(std::make_shared(index_shape, const_0, const_1, const_1)); + + // [A, B, ..., T, ..., K] --> [A, B, ..., len(index), ..., K] + auto target_shape = std::make_shared(input_shape, + dim_vec, + index_len, + v0::Constant::create(element::i32, Shape{}, {0})); + + // broadcast && index fill + auto broadcasted_value = context.mark_node(std::make_shared(value_vec, target_shape, dim_vec)); + auto broadcasted_index = context.mark_node(std::make_shared(index, target_shape, dim_vec)); + auto result = context.mark_node( + std::make_shared(input, broadcasted_index, broadcasted_value, dim)); + + return {result}; +}; + +} // namespace op +} // namespace pytorch +} // namespace frontend +} // namespace ov diff --git a/src/frontends/pytorch/src/op_table.cpp b/src/frontends/pytorch/src/op_table.cpp index 0fd2d8e54006a4..7307833430411f 100644 --- a/src/frontends/pytorch/src/op_table.cpp +++ b/src/frontends/pytorch/src/op_table.cpp @@ -113,6 +113,7 @@ OP_CONVERTER(translate_im2col); OP_CONVERTER(translate_index); OP_CONVERTER(translate_index_add); OP_CONVERTER(translate_index_copy_); +OP_CONVERTER(translate_index_fill_); OP_CONVERTER(translate_index_put_); OP_CONVERTER(translate_index_select); OP_CONVERTER(translate_instance_norm); @@ -496,6 +497,7 @@ const std::unordered_map get_supported_ops_ts() { // aten::imag - Supported in limited set of patterns // aten::index - Supported in limited set of patterns {"aten::index_copy_", op::inplace_op}, + {"aten::index_fill_", op::inplace_op}, {"aten::index_put_", op::inplace_op}, {"aten::index_add", op::translate_index_add}, {"aten::index_select", op::translate_index_select}, diff --git a/src/inference/include/openvino/runtime/infer_request.hpp b/src/inference/include/openvino/runtime/infer_request.hpp index ed4dcd67797b84..10a606a2b6c535 100644 --- a/src/inference/include/openvino/runtime/infer_request.hpp +++ b/src/inference/include/openvino/runtime/infer_request.hpp @@ -255,7 +255,7 @@ class OPENVINO_RUNTIME_API InferRequest { /** * @brief Infers specified input(s) in synchronous mode. * @note It blocks all methods of InferRequest while request is ongoing (running or waiting in a queue). - * Calling any method leads to throwning the ov::Busy exception. + * Calling any method leads to throwing the ov::Busy exception. */ void infer(); @@ -274,7 +274,7 @@ class OPENVINO_RUNTIME_API InferRequest { /** * @brief Starts inference of specified input(s) in asynchronous mode. * @note It returns immediately. Inference starts also immediately. - * Calling any method while the request in a running state leads to throwning the ov::Busy exception. + * Calling any method while the request in a running state leads to throwing the ov::Busy exception. */ void start_async(); diff --git a/src/inference/include/openvino/runtime/intel_npu/properties.hpp b/src/inference/include/openvino/runtime/intel_npu/properties.hpp index 49416f61b8b43b..8734757da1d53d 100644 --- a/src/inference/include/openvino/runtime/intel_npu/properties.hpp +++ b/src/inference/include/openvino/runtime/intel_npu/properties.hpp @@ -95,5 +95,12 @@ static constexpr ov::Property max_tiles{"NPU_MAX_TILES"}; */ static constexpr ov::Property bypass_umd_caching{"NPU_BYPASS_UMD_CACHING"}; +/** + * @brief [Only for NPU Plugin] + * Type: boolean, default is false + * This option allows to delay loading the weights until inference is created + */ +static constexpr ov::Property defer_weights_load{"NPU_DEFER_WEIGHTS_LOAD"}; + } // namespace intel_npu } // namespace ov diff --git a/src/plugins/intel_cpu/docs/debug_capabilities/README.md b/src/plugins/intel_cpu/docs/debug_capabilities/README.md index f7989217be4a7f..dee46c2b0af9b5 100644 --- a/src/plugins/intel_cpu/docs/debug_capabilities/README.md +++ b/src/plugins/intel_cpu/docs/debug_capabilities/README.md @@ -24,3 +24,5 @@ Use the following cmake option to enable debug capabilities: `OV_CPU_SUMMARY_PERF=1` Set the environment variable to display performance summary at the time when model is being destructed. Internal performance counter will be enabled automatically. +* [Average counters](average_counters.md) + `OV_CPU_AVERAGE_COUNTERS=filename` diff --git a/src/plugins/intel_cpu/docs/debug_capabilities/average_counters.md b/src/plugins/intel_cpu/docs/debug_capabilities/average_counters.md new file mode 100644 index 00000000000000..921c117d0edb01 --- /dev/null +++ b/src/plugins/intel_cpu/docs/debug_capabilities/average_counters.md @@ -0,0 +1,17 @@ +# Average counters + +To enable collection of per-node average counters the following environment variable should be used: + +```sh + OV_CPU_AVERAGE_COUNTERS= binary ... +``` + +The output table has the same format as: + +```sh + benchmark_app --report_type average_counters +``` + +The avarage counters can be aggregated using: + +* [aggregate-average-counters.py](../../tools/aggregate-average-counters/aggregate-average-counters.py) diff --git a/src/plugins/intel_cpu/src/config.cpp b/src/plugins/intel_cpu/src/config.cpp index 92470ca063a4c0..7ce4c1069e695d 100644 --- a/src/plugins/intel_cpu/src/config.cpp +++ b/src/plugins/intel_cpu/src/config.cpp @@ -60,9 +60,12 @@ Config::Config() { * configuration properties */ void Config::applyDebugCapsProperties() { - // always enable perf counters for verbose mode and performance summary - if (!debugCaps.verbose.empty() || !debugCaps.summaryPerf.empty()) + // always enable perf counters for verbose, performance summary and average counters + if (!debugCaps.verbose.empty() || + !debugCaps.summaryPerf.empty() || + !debugCaps.averageCountersPath.empty()) { collectPerfCounters = true; + } } #endif diff --git a/src/plugins/intel_cpu/src/emitters/plugin/aarch64/jit_eltwise_emitters.cpp b/src/plugins/intel_cpu/src/emitters/plugin/aarch64/jit_eltwise_emitters.cpp index 355c8fb7f4c4d7..4aec56d98873fa 100644 --- a/src/plugins/intel_cpu/src/emitters/plugin/aarch64/jit_eltwise_emitters.cpp +++ b/src/plugins/intel_cpu/src/emitters/plugin/aarch64/jit_eltwise_emitters.cpp @@ -516,6 +516,51 @@ std::set> jit_floor_emitter::get_supported_precisions return {{element::f32}}; } +/// FLOOR_MOD /// +jit_floor_mod_emitter::jit_floor_mod_emitter(dnnl::impl::cpu::aarch64::jit_generator *host, + dnnl::impl::cpu::aarch64::cpu_isa_t host_isa, + const std::shared_ptr& node) + : jit_emitter(host, host_isa, node, get_arithmetic_binary_exec_precision(node)) { +} + +jit_floor_mod_emitter::jit_floor_mod_emitter(dnnl::impl::cpu::aarch64::jit_generator *host, + dnnl::impl::cpu::aarch64::cpu_isa_t host_isa, + const ov::element::Type exec_prc): jit_emitter(host, host_isa, exec_prc) { +} + +size_t jit_floor_mod_emitter::get_inputs_count() const { return 2; } + +size_t jit_floor_mod_emitter::get_aux_vecs_count() const { return 1; } + +void jit_floor_mod_emitter::emit_impl(const std::vector &in_vec_idxs, const std::vector &out_vec_idxs) const { + if (host_isa_ == dnnl::impl::cpu::aarch64::asimd) { + emit_isa(in_vec_idxs, out_vec_idxs); + } else { + OV_CPU_JIT_EMITTER_THROW("Can't create jit eltwise kernel"); + } +} + +template +void jit_floor_mod_emitter::emit_isa(const std::vector &in_vec_idxs, const std::vector &out_vec_idxs) const { + OV_CPU_JIT_EMITTER_ASSERT(exec_prc_ == ov::element::f32, "unsupported precision: " + exec_prc_.to_string()); + + using TReg = typename dnnl::impl::cpu::aarch64::cpu_isa_traits::TReg; + + TReg dividend = TReg(in_vec_idxs[0]); + TReg divisor = TReg(in_vec_idxs[1]); + TReg r = TReg(out_vec_idxs[0]); + TReg aux = TReg(aux_vec_idxs[0]); + + h->fdiv(aux.s, dividend.s, divisor.s); + h->frintm(aux.s, aux.s); + h->fmul(aux.s, aux.s, divisor.s); + h->fsub(r.s, dividend.s, aux.s); +} + +std::set> jit_floor_mod_emitter::get_supported_precisions(const std::shared_ptr& node) { + return {{element::f32, element::f32}}; +} + /// CEILING /// //Initialization of the emitter, taking node as input jit_ceiling_emitter::jit_ceiling_emitter(dnnl::impl::cpu::aarch64::jit_generator* host, @@ -1232,6 +1277,58 @@ std::set> jit_less_equal_emitter::get_supported_preci return {{element::f32, element::f32}}; } +/// LOGICAL_AND /// +jit_logical_and_emitter::jit_logical_and_emitter(dnnl::impl::cpu::aarch64::jit_generator* host, + dnnl::impl::cpu::aarch64::cpu_isa_t host_isa, + const std::shared_ptr& node) + : jit_emitter(host, host_isa, node, get_arithmetic_binary_exec_precision(node)) { + prepare_table(); +} + +jit_logical_and_emitter::jit_logical_and_emitter(dnnl::impl::cpu::aarch64::jit_generator* host, + dnnl::impl::cpu::aarch64::cpu_isa_t host_isa, + const ov::element::Type exec_prc) + : jit_emitter(host, host_isa, exec_prc) { + prepare_table(); +} + +size_t jit_logical_and_emitter::get_inputs_count() const { return 2; } + +size_t jit_logical_and_emitter::get_aux_vecs_count() const { return 1; } + +size_t jit_logical_and_emitter::get_aux_gprs_count() const { return 1; } + +void jit_logical_and_emitter::emit_impl(const std::vector &in_vec_idxs, const std::vector &out_vec_idxs) const { + if (host_isa_ == dnnl::impl::cpu::aarch64::asimd) { + emit_isa(in_vec_idxs, out_vec_idxs); + } else { + OV_CPU_JIT_EMITTER_THROW("Can't create jit eltwise kernel"); + } +} + +template +void jit_logical_and_emitter::emit_isa(const std::vector &in_vec_idxs, const std::vector &out_vec_idxs) const { + OV_CPU_JIT_EMITTER_ASSERT(exec_prc_ == ov::element::f32, "unsupported precision: " + exec_prc_.to_string()); + + using TReg = typename dnnl::impl::cpu::aarch64::cpu_isa_traits::TReg; + const TReg src1 = TReg(in_vec_idxs[0]); + const TReg src2 = TReg(in_vec_idxs[1]); + const TReg dst = TReg(out_vec_idxs[0]); + const TReg aux = TReg(aux_vec_idxs[0]); + + h->and_(dst.b16, src1.b16, src2.b16); + h->ld1r(aux.s, table_val2("one")); + h->and_(dst.b16, dst.b16, aux.b16); +} + +void jit_logical_and_emitter::register_table_entries() { + push_arg_entry_of("one", 0x3f800000, true); +} + +std::set> jit_logical_and_emitter::get_supported_precisions(const std::shared_ptr& node) { + return {{element::f32, element::f32}}; +} + /// LOGICAL_NOT /// jit_logical_not_emitter::jit_logical_not_emitter(dnnl::impl::cpu::aarch64::jit_generator* host, dnnl::impl::cpu::aarch64::cpu_isa_t host_isa, diff --git a/src/plugins/intel_cpu/src/emitters/plugin/aarch64/jit_eltwise_emitters.hpp b/src/plugins/intel_cpu/src/emitters/plugin/aarch64/jit_eltwise_emitters.hpp index a99e016c9c834a..2cb7e6928ade3e 100644 --- a/src/plugins/intel_cpu/src/emitters/plugin/aarch64/jit_eltwise_emitters.hpp +++ b/src/plugins/intel_cpu/src/emitters/plugin/aarch64/jit_eltwise_emitters.hpp @@ -213,7 +213,28 @@ class jit_floor_emitter : public jit_emitter { template void emit_isa(const std::vector &in_vec_idxs, const std::vector &out_vec_idxs) const; }; +class jit_floor_mod_emitter : public jit_emitter { +public: + jit_floor_mod_emitter(dnnl::impl::cpu::aarch64::jit_generator *host, + dnnl::impl::cpu::aarch64::cpu_isa_t host_isa, + const ov::element::Type exec_prc = ov::element::f32); + + jit_floor_mod_emitter(dnnl::impl::cpu::aarch64::jit_generator *host, + dnnl::impl::cpu::aarch64::cpu_isa_t host_isa, + const std::shared_ptr& node); + + size_t get_inputs_count() const override; + + size_t get_aux_vecs_count() const override; + + static std::set> get_supported_precisions(const std::shared_ptr& node = nullptr); + +private: + void emit_impl(const std::vector &in_vec_idxs, const std::vector &out_vec_idxs) const override; + template + void emit_isa(const std::vector &in_vec_idxs, const std::vector &out_vec_idxs) const; +}; class jit_ceiling_emitter : public jit_emitter { public: // Constructor with explicit precision @@ -579,6 +600,34 @@ class jit_less_equal_emitter : public jit_emitter { void register_table_entries() override; }; +class jit_logical_and_emitter : public jit_emitter { +public: + jit_logical_and_emitter(dnnl::impl::cpu::aarch64::jit_generator *host, + dnnl::impl::cpu::aarch64::cpu_isa_t host_isa, + const ov::element::Type exec_prc = ov::element::f32); + + jit_logical_and_emitter(dnnl::impl::cpu::aarch64::jit_generator *host, + dnnl::impl::cpu::aarch64::cpu_isa_t host_isa, + const std::shared_ptr& n); + + size_t get_inputs_count() const override; + + size_t get_aux_vecs_count() const override; + + size_t get_aux_gprs_count() const override; + + static std::set> get_supported_precisions( + const std::shared_ptr& node = nullptr); + +private: + void emit_impl(const std::vector &in_vec_idxs, const std::vector &out_vec_idxs) const override; + + template + void emit_isa(const std::vector &in_vec_idxs, const std::vector &out_vec_idxs) const; + + void register_table_entries() override; +}; + class jit_logical_not_emitter : public jit_emitter { public: jit_logical_not_emitter(dnnl::impl::cpu::aarch64::jit_generator* host, diff --git a/src/plugins/intel_cpu/src/graph.cpp b/src/plugins/intel_cpu/src/graph.cpp index 86fb7bf4ded0a1..6aa4644f902bc9 100644 --- a/src/plugins/intel_cpu/src/graph.cpp +++ b/src/plugins/intel_cpu/src/graph.cpp @@ -58,6 +58,7 @@ namespace intel_cpu { Graph::~Graph() { CPU_DEBUG_CAP_ENABLE(summary_perf(*this)); + CPU_DEBUG_CAP_ENABLE(average_counters(*this)); } template diff --git a/src/plugins/intel_cpu/src/graph_dumper.cpp b/src/plugins/intel_cpu/src/graph_dumper.cpp index fab6e99dcf2550..04c15408743c71 100644 --- a/src/plugins/intel_cpu/src/graph_dumper.cpp +++ b/src/plugins/intel_cpu/src/graph_dumper.cpp @@ -10,10 +10,12 @@ #include "openvino/runtime/exec_model_info.hpp" #include "utils/debug_capabilities.h" +#include #include #include #include #include +#include namespace ov { namespace intel_cpu { @@ -346,6 +348,62 @@ void summary_perf(const Graph &graph) { } } +void average_counters(const Graph &graph) { + /** + * @todo improve logic for a graph with inner graphs: + * - collect counters only for the outer graph if full path is specified + * - collect counters for all the graphs if some keyword (i.e. 'all') is specified, using the following form: + * - _.csv + * For example: 0_MyModel.csv + */ + static int graphIndex = 0; + + std::ofstream file; + std::string fileName = graph.getConfig().debugCaps.averageCountersPath + "_" + std::to_string(graphIndex++) + ".csv"; + + file.open(fileName); + + // table structure is identical to the benchmark_app average_counters report + const std::string header = "layerName;execStatus;layerType;execType;realTime (ms);cpuTime (ms);"; + file << header << "\n"; + + uint64_t total = 0; + + auto toMs = [](uint64_t value) { + return std::chrono::microseconds(value).count() / 1000.0; + }; + + auto printAverageCounter = [&toMs, &file](NodePtr node) { + const uint64_t avg = node->PerfCounter().avg(); + const std::string status = avg > 0 ? "EXECUTED" : "NOT_RUN"; + const auto cpuTime = toMs(avg); + const auto realTime = cpuTime; + + file << node->getName() << ";" + << status << ";" + << node->getTypeStr() << ";" + << node->getPrimitiveDescriptorType() << ";" + << realTime << ";" + << cpuTime << ";" + << "\n"; + + return avg; + }; + + for (auto &node : graph.GetNodes()) { + if (node->isConstant()) + continue; + + total += printAverageCounter(node); + } + + const auto totalMs = toMs(total); + + file << "Total;;;;" << totalMs << ";" << totalMs << ";" << "\n"; + + file.close(); +} + #endif } // namespace intel_cpu } // namespace ov diff --git a/src/plugins/intel_cpu/src/graph_dumper.h b/src/plugins/intel_cpu/src/graph_dumper.h index b4a94a31d644cc..417db7e4c3cdc5 100644 --- a/src/plugins/intel_cpu/src/graph_dumper.h +++ b/src/plugins/intel_cpu/src/graph_dumper.h @@ -15,6 +15,7 @@ std::shared_ptr dump_graph_as_ie_ngraph_net(const Graph &graph); #ifdef CPU_DEBUG_CAPS void serialize(const Graph &graph); void summary_perf(const Graph &graph); +void average_counters(const Graph &graph); #endif // CPU_DEBUG_CAPS } // namespace intel_cpu diff --git a/src/plugins/intel_cpu/src/nodes/batch_to_space.cpp b/src/plugins/intel_cpu/src/nodes/batch_to_space.cpp index 8ec1fdab85e1ed..80713e90750e2d 100644 --- a/src/plugins/intel_cpu/src/nodes/batch_to_space.cpp +++ b/src/plugins/intel_cpu/src/nodes/batch_to_space.cpp @@ -29,7 +29,7 @@ bool BatchToSpace::isSupportedOperation(const std::shared_ptr& o } BatchToSpace::BatchToSpace(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, PortMask(1, 2, 3))) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/bin_conv.cpp b/src/plugins/intel_cpu/src/nodes/bin_conv.cpp index f2de2d20a17528..d1e82235ba9bb1 100644 --- a/src/plugins/intel_cpu/src/nodes/bin_conv.cpp +++ b/src/plugins/intel_cpu/src/nodes/bin_conv.cpp @@ -899,7 +899,7 @@ bool BinaryConvolution::isSupportedOperation(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (isSupportedOperation(op, errorMessage)) { errorPrefix = "BinaryConvolution node with name '" + getName() + "' "; diff --git a/src/plugins/intel_cpu/src/nodes/broadcast.cpp b/src/plugins/intel_cpu/src/nodes/broadcast.cpp index ac8dd814ae9961..c88803e07de601 100644 --- a/src/plugins/intel_cpu/src/nodes/broadcast.cpp +++ b/src/plugins/intel_cpu/src/nodes/broadcast.cpp @@ -47,7 +47,7 @@ bool Broadcast::isSupportedOperation(const std::shared_ptr& op, } Broadcast::Broadcast(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, PortMask(TARGET_SHAPE_IDX, AXES_MAPPING_IDX))) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/col2im.cpp b/src/plugins/intel_cpu/src/nodes/col2im.cpp index 7faa49206130ca..4b83e78fd82505 100644 --- a/src/plugins/intel_cpu/src/nodes/col2im.cpp +++ b/src/plugins/intel_cpu/src/nodes/col2im.cpp @@ -10,7 +10,7 @@ namespace ov { namespace intel_cpu { namespace node { Col2Im::Col2Im(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, PortMask(1, 2))) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/concat.cpp b/src/plugins/intel_cpu/src/nodes/concat.cpp index ed8e34328fb8a3..635f37b2d05b3a 100644 --- a/src/plugins/intel_cpu/src/nodes/concat.cpp +++ b/src/plugins/intel_cpu/src/nodes/concat.cpp @@ -50,7 +50,7 @@ bool Concat::isSupportedOperation(const std::shared_ptr& op, std } Concat::Concat(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/conv.cpp b/src/plugins/intel_cpu/src/nodes/conv.cpp index cbdb35db271622..7cf7698e989343 100644 --- a/src/plugins/intel_cpu/src/nodes/conv.cpp +++ b/src/plugins/intel_cpu/src/nodes/conv.cpp @@ -240,7 +240,7 @@ bool Convolution::isSupportedOperation(const std::shared_ptr& op } Convolution::Convolution(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)), withBiases(false), withSum(false), withDWConv(false), + : Node(op, context, NgraphShapeInferFactory(op)), withBiases(false), withSum(false), withDWConv(false), isGrouped(false), dw_conv_oc(0), dw_conv_ih(0), dw_conv_iw(0), dw_conv_in_dt(memory::data_type::undef), groupNum(1lu), IC(1), groupIC(1), groupOC(1), eltwisePrecision(ov::element::f32) { std::string errorMessage; diff --git a/src/plugins/intel_cpu/src/nodes/ctc_greedy_decoder.cpp b/src/plugins/intel_cpu/src/nodes/ctc_greedy_decoder.cpp index 85c41acedf6035..0b467fe452e061 100644 --- a/src/plugins/intel_cpu/src/nodes/ctc_greedy_decoder.cpp +++ b/src/plugins/intel_cpu/src/nodes/ctc_greedy_decoder.cpp @@ -27,7 +27,7 @@ bool CTCGreedyDecoder::isSupportedOperation(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/ctc_greedy_decoder_seq_len.cpp b/src/plugins/intel_cpu/src/nodes/ctc_greedy_decoder_seq_len.cpp index be343076281a81..63db3968094c3a 100644 --- a/src/plugins/intel_cpu/src/nodes/ctc_greedy_decoder_seq_len.cpp +++ b/src/plugins/intel_cpu/src/nodes/ctc_greedy_decoder_seq_len.cpp @@ -27,7 +27,7 @@ bool CTCGreedyDecoderSeqLen::isSupportedOperation(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/ctc_loss.cpp b/src/plugins/intel_cpu/src/nodes/ctc_loss.cpp index 3161c9a0e87a84..6d09b0aea7e934 100644 --- a/src/plugins/intel_cpu/src/nodes/ctc_loss.cpp +++ b/src/plugins/intel_cpu/src/nodes/ctc_loss.cpp @@ -26,7 +26,7 @@ bool CTCLoss::isSupportedOperation(const std::shared_ptr& op, st } CTCLoss::CTCLoss(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/cum_sum.cpp b/src/plugins/intel_cpu/src/nodes/cum_sum.cpp index 8327084839cc14..e411283e661585 100644 --- a/src/plugins/intel_cpu/src/nodes/cum_sum.cpp +++ b/src/plugins/intel_cpu/src/nodes/cum_sum.cpp @@ -29,7 +29,8 @@ bool CumSum::isSupportedOperation(const std::shared_ptr& op, std return true; } -CumSum::CumSum(const std::shared_ptr& op, const GraphContext::CPtr context) : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { +CumSum::CumSum(const std::shared_ptr& op, const GraphContext::CPtr context) + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/def_conv.cpp b/src/plugins/intel_cpu/src/nodes/def_conv.cpp index eb56902c653e99..0167a18673c444 100644 --- a/src/plugins/intel_cpu/src/nodes/def_conv.cpp +++ b/src/plugins/intel_cpu/src/nodes/def_conv.cpp @@ -745,7 +745,7 @@ bool DefConvKey::operator==(const DefConvKey &rhs) const { } // namespace DeformableConvolution::DeformableConvolution(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/depth_to_space.cpp b/src/plugins/intel_cpu/src/nodes/depth_to_space.cpp index 15d521a423c7e7..eb3789068adca1 100644 --- a/src/plugins/intel_cpu/src/nodes/depth_to_space.cpp +++ b/src/plugins/intel_cpu/src/nodes/depth_to_space.cpp @@ -67,7 +67,7 @@ bool DepthToSpace::isSupportedOperation(const std::shared_ptr& o } DepthToSpace::DepthToSpace(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/detection_output.cpp b/src/plugins/intel_cpu/src/nodes/detection_output.cpp index cb9ec41ba4f70b..99702780b83034 100644 --- a/src/plugins/intel_cpu/src/nodes/detection_output.cpp +++ b/src/plugins/intel_cpu/src/nodes/detection_output.cpp @@ -49,7 +49,7 @@ bool DetectionOutput::isSupportedOperation(const std::shared_ptr } DetectionOutput::DetectionOutput(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/dft.cpp b/src/plugins/intel_cpu/src/nodes/dft.cpp index 32ec9ef9630518..76ecbbb36617f5 100644 --- a/src/plugins/intel_cpu/src/nodes/dft.cpp +++ b/src/plugins/intel_cpu/src/nodes/dft.cpp @@ -42,8 +42,8 @@ bool DFT::isSupportedOperation(const std::shared_ptr& op, std::s return true; } -DFT::DFT(const std::shared_ptr& op, const GraphContext::CPtr context) : - Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { +DFT::DFT(const std::shared_ptr& op, const GraphContext::CPtr context) + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/embedding_bag_offsets.cpp b/src/plugins/intel_cpu/src/nodes/embedding_bag_offsets.cpp index dc69892dabb2e4..b5fbaee982808d 100644 --- a/src/plugins/intel_cpu/src/nodes/embedding_bag_offsets.cpp +++ b/src/plugins/intel_cpu/src/nodes/embedding_bag_offsets.cpp @@ -29,7 +29,7 @@ bool EmbeddingBagOffset::isSupportedOperation(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)), + : Node(op, context, NgraphShapeInferFactory(op)), EmbeddingBag(op, 3lu, 1lu, 4lu, 3lu) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { diff --git a/src/plugins/intel_cpu/src/nodes/embedding_bag_packed.cpp b/src/plugins/intel_cpu/src/nodes/embedding_bag_packed.cpp index 0b490a28a81487..fd2e0b6141f1fc 100644 --- a/src/plugins/intel_cpu/src/nodes/embedding_bag_packed.cpp +++ b/src/plugins/intel_cpu/src/nodes/embedding_bag_packed.cpp @@ -28,7 +28,7 @@ bool EmbeddingBagPacked::isSupportedOperation(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)), + : Node(op, context, NgraphShapeInferFactory(op)), EmbeddingBag(op, 2lu, 1lu, 2lu, 3lu) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { diff --git a/src/plugins/intel_cpu/src/nodes/embedding_segments_sum.cpp b/src/plugins/intel_cpu/src/nodes/embedding_segments_sum.cpp index 1b2e28e6039543..2a012c6b941831 100644 --- a/src/plugins/intel_cpu/src/nodes/embedding_segments_sum.cpp +++ b/src/plugins/intel_cpu/src/nodes/embedding_segments_sum.cpp @@ -26,7 +26,7 @@ bool EmbeddingSegmentsSum::isSupportedOperation(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, PortMask(NUM_SEGMENTS_IDX))), + : Node(op, context, NgraphShapeInferFactory(op)), EmbeddingBag(op, 4lu, 1lu, 5lu, 4lu) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { diff --git a/src/plugins/intel_cpu/src/nodes/executors/aarch64/jit_eltwise.cpp b/src/plugins/intel_cpu/src/nodes/executors/aarch64/jit_eltwise.cpp index 6da6b63eb94a72..5f63904fbb9342 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/aarch64/jit_eltwise.cpp +++ b/src/plugins/intel_cpu/src/nodes/executors/aarch64/jit_eltwise.cpp @@ -26,6 +26,7 @@ bool JitEltwiseExecutor::isSupported( Algorithm::EltwiseEqual, Algorithm::EltwiseExp, Algorithm::EltwiseFloor, + Algorithm::EltwiseFloorMod, Algorithm::EltwiseCeiling, Algorithm::EltwiseGeluErf, Algorithm::EltwiseGeluTanh, @@ -36,6 +37,7 @@ bool JitEltwiseExecutor::isSupported( Algorithm::EltwiseIsInf, Algorithm::EltwiseIsNaN, Algorithm::EltwiseLessEqual, + Algorithm::EltwiseLogicalAnd, Algorithm::EltwiseLogicalNot, Algorithm::EltwiseLogicalXor, Algorithm::EltwiseMaximum, diff --git a/src/plugins/intel_cpu/src/nodes/experimental_detectron_detection_output.cpp b/src/plugins/intel_cpu/src/nodes/experimental_detectron_detection_output.cpp index cb21606638c750..441e013af2cbbf 100644 --- a/src/plugins/intel_cpu/src/nodes/experimental_detectron_detection_output.cpp +++ b/src/plugins/intel_cpu/src/nodes/experimental_detectron_detection_output.cpp @@ -236,7 +236,7 @@ bool ExperimentalDetectronDetectionOutput::isSupportedOperation(const std::share ExperimentalDetectronDetectionOutput::ExperimentalDetectronDetectionOutput(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/experimental_detectron_generate_proposals_single_image.cpp b/src/plugins/intel_cpu/src/nodes/experimental_detectron_generate_proposals_single_image.cpp index 26adc6fc573aa3..33f17c8d95f093 100644 --- a/src/plugins/intel_cpu/src/nodes/experimental_detectron_generate_proposals_single_image.cpp +++ b/src/plugins/intel_cpu/src/nodes/experimental_detectron_generate_proposals_single_image.cpp @@ -286,7 +286,7 @@ bool ExperimentalDetectronGenerateProposalsSingleImage::isSupportedOperation ExperimentalDetectronGenerateProposalsSingleImage::ExperimentalDetectronGenerateProposalsSingleImage( const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/experimental_detectron_priorgridgenerator.cpp b/src/plugins/intel_cpu/src/nodes/experimental_detectron_priorgridgenerator.cpp index dc01230ee50623..eead95def0a8fb 100644 --- a/src/plugins/intel_cpu/src/nodes/experimental_detectron_priorgridgenerator.cpp +++ b/src/plugins/intel_cpu/src/nodes/experimental_detectron_priorgridgenerator.cpp @@ -26,10 +26,9 @@ bool ExperimentalDetectronPriorGridGenerator::isSupportedOperation(const std::sh return true; } -ExperimentalDetectronPriorGridGenerator::ExperimentalDetectronPriorGridGenerator( - const std::shared_ptr& op, - const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { +ExperimentalDetectronPriorGridGenerator::ExperimentalDetectronPriorGridGenerator(const std::shared_ptr& op, + const GraphContext::CPtr context) + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/experimental_detectron_roifeatureextractor.cpp b/src/plugins/intel_cpu/src/nodes/experimental_detectron_roifeatureextractor.cpp index a94337145eb092..c92e3c2594d4a9 100644 --- a/src/plugins/intel_cpu/src/nodes/experimental_detectron_roifeatureextractor.cpp +++ b/src/plugins/intel_cpu/src/nodes/experimental_detectron_roifeatureextractor.cpp @@ -293,10 +293,9 @@ bool ExperimentalDetectronROIFeatureExtractor::isSupportedOperation(const std::s return true; } -ExperimentalDetectronROIFeatureExtractor::ExperimentalDetectronROIFeatureExtractor( - const std::shared_ptr& op, - const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { +ExperimentalDetectronROIFeatureExtractor::ExperimentalDetectronROIFeatureExtractor(const std::shared_ptr& op, + const GraphContext::CPtr context) + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/experimental_detectron_topkrois.cpp b/src/plugins/intel_cpu/src/nodes/experimental_detectron_topkrois.cpp index ac778b0d4d91ce..46b60fcdb83efd 100644 --- a/src/plugins/intel_cpu/src/nodes/experimental_detectron_topkrois.cpp +++ b/src/plugins/intel_cpu/src/nodes/experimental_detectron_topkrois.cpp @@ -30,7 +30,7 @@ bool ExperimentalDetectronTopKROIs::isSupportedOperation(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/extract_image_patches.cpp b/src/plugins/intel_cpu/src/nodes/extract_image_patches.cpp index 7c3dda49931451..8b5d0b510614e1 100644 --- a/src/plugins/intel_cpu/src/nodes/extract_image_patches.cpp +++ b/src/plugins/intel_cpu/src/nodes/extract_image_patches.cpp @@ -329,7 +329,7 @@ bool ExtractImagePatchesKey::operator==(const ExtractImagePatchesKey& rhs) const } // namespace ExtractImagePatches::ExtractImagePatches(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/gather_elements.cpp b/src/plugins/intel_cpu/src/nodes/gather_elements.cpp index f0fd14121b2c67..8653bda8c483d3 100644 --- a/src/plugins/intel_cpu/src/nodes/gather_elements.cpp +++ b/src/plugins/intel_cpu/src/nodes/gather_elements.cpp @@ -30,7 +30,7 @@ bool GatherElements::isSupportedOperation(const std::shared_ptr& } GatherElements::GatherElements(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/gather_nd.cpp b/src/plugins/intel_cpu/src/nodes/gather_nd.cpp index 2dc91dd12559f2..8c81f9b770a687 100644 --- a/src/plugins/intel_cpu/src/nodes/gather_nd.cpp +++ b/src/plugins/intel_cpu/src/nodes/gather_nd.cpp @@ -32,7 +32,7 @@ bool GatherND::isSupportedOperation(const std::shared_ptr& op, s } GatherND::GatherND(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/gather_tree.cpp b/src/plugins/intel_cpu/src/nodes/gather_tree.cpp index f318290defbf82..5834cd1e1048ba 100644 --- a/src/plugins/intel_cpu/src/nodes/gather_tree.cpp +++ b/src/plugins/intel_cpu/src/nodes/gather_tree.cpp @@ -29,7 +29,7 @@ bool GatherTree::isSupportedOperation(const std::shared_ptr& op, } GatherTree::GatherTree(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/grid_sample.cpp b/src/plugins/intel_cpu/src/nodes/grid_sample.cpp index c8eed21bb312f5..c8b73360539b68 100644 --- a/src/plugins/intel_cpu/src/nodes/grid_sample.cpp +++ b/src/plugins/intel_cpu/src/nodes/grid_sample.cpp @@ -39,7 +39,7 @@ bool GridSample::isSupportedOperation(const std::shared_ptr& op, #if defined(OPENVINO_ARCH_X86_64) GridSample::GridSample(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, PortMask(1))) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { THROW_CPU_NODE_ERR(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/grn.cpp b/src/plugins/intel_cpu/src/nodes/grn.cpp index f20e7d6b90a012..10de2ef2286f0f 100644 --- a/src/plugins/intel_cpu/src/nodes/grn.cpp +++ b/src/plugins/intel_cpu/src/nodes/grn.cpp @@ -26,7 +26,7 @@ bool GRN::isSupportedOperation(const std::shared_ptr& op, std::s } GRN::GRN(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/interaction.cpp b/src/plugins/intel_cpu/src/nodes/interaction.cpp index 6f604f4a9e278a..5ec48e7a263272 100644 --- a/src/plugins/intel_cpu/src/nodes/interaction.cpp +++ b/src/plugins/intel_cpu/src/nodes/interaction.cpp @@ -164,7 +164,7 @@ struct jit_move_scale_kernel : public jit_uni_move_scale_kernel, public jit_gene #endif // OPENVINO_ARCH_X86_64 Interaction::Interaction(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/inverse.cpp b/src/plugins/intel_cpu/src/nodes/inverse.cpp index 6eeecfad7222a9..bff23846966a9d 100644 --- a/src/plugins/intel_cpu/src/nodes/inverse.cpp +++ b/src/plugins/intel_cpu/src/nodes/inverse.cpp @@ -16,7 +16,7 @@ namespace intel_cpu { namespace node { Inverse::Inverse(const std::shared_ptr& op, const GraphContext::CPtr& context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { THROW_CPU_NODE_ERR(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/kernels/aarch64/jit_uni_eltwise_generic.cpp b/src/plugins/intel_cpu/src/nodes/kernels/aarch64/jit_uni_eltwise_generic.cpp index b7fbfaf16e1587..9a1662e2c5dab5 100644 --- a/src/plugins/intel_cpu/src/nodes/kernels/aarch64/jit_uni_eltwise_generic.cpp +++ b/src/plugins/intel_cpu/src/nodes/kernels/aarch64/jit_uni_eltwise_generic.cpp @@ -648,11 +648,13 @@ std::shared_ptr jit_uni_eltwise_generic::create_eltwise_emitte OV_CASE(Algorithm::EltwiseEqual, ov::intel_cpu::aarch64::jit_equal_emitter), OV_CASE(Algorithm::EltwiseExp, ov::intel_cpu::aarch64::jit_exp_emitter), OV_CASE(Algorithm::EltwiseFloor, ov::intel_cpu::aarch64::jit_floor_emitter), + OV_CASE(Algorithm::EltwiseFloorMod, ov::intel_cpu::aarch64::jit_floor_mod_emitter), OV_CASE(Algorithm::EltwiseCeiling, ov::intel_cpu::aarch64::jit_ceiling_emitter), OV_CASE(Algorithm::EltwiseHswish, ov::intel_cpu::aarch64::jit_hswish_emitter), OV_CASE(Algorithm::EltwiseIsFinite, ov::intel_cpu::aarch64::jit_is_finite_emitter), OV_CASE(Algorithm::EltwiseIsInf, ov::intel_cpu::aarch64::jit_is_inf_emitter), OV_CASE(Algorithm::EltwiseLessEqual, ov::intel_cpu::aarch64::jit_less_equal_emitter), + OV_CASE(Algorithm::EltwiseLogicalAnd, ov::intel_cpu::aarch64::jit_logical_and_emitter), OV_CASE(Algorithm::EltwiseLogicalNot, ov::intel_cpu::aarch64::jit_logical_not_emitter), OV_CASE(Algorithm::EltwiseLogicalXor, ov::intel_cpu::aarch64::jit_logical_xor_emitter), OV_CASE(Algorithm::EltwiseIsNaN, ov::intel_cpu::aarch64::jit_is_nan_emitter), @@ -829,6 +831,7 @@ std::set> eltwise_precision_helper::get_supported_pre OV_CASE(Algorithm::EltwiseEqual, jit_equal_emitter), OV_CASE(Algorithm::EltwiseExp, jit_exp_emitter), OV_CASE(Algorithm::EltwiseFloor, jit_floor_emitter), + OV_CASE(Algorithm::EltwiseFloorMod, jit_floor_mod_emitter), OV_CASE(Algorithm::EltwiseCeiling, jit_ceiling_emitter), OV_CASE(Algorithm::EltwiseGeluErf, jit_gelu_erf_emitter), OV_CASE(Algorithm::EltwiseGeluTanh, jit_gelu_tanh_emitter), @@ -839,6 +842,7 @@ std::set> eltwise_precision_helper::get_supported_pre OV_CASE(Algorithm::EltwiseIsInf, jit_is_inf_emitter), OV_CASE(Algorithm::EltwiseIsNaN, jit_is_nan_emitter), OV_CASE(Algorithm::EltwiseLessEqual, jit_less_equal_emitter), + OV_CASE(Algorithm::EltwiseLogicalAnd, jit_logical_and_emitter), OV_CASE(Algorithm::EltwiseLogicalNot, jit_logical_not_emitter), OV_CASE(Algorithm::EltwiseLogicalXor, jit_logical_xor_emitter), OV_CASE(Algorithm::EltwiseMaximum, jit_maximum_emitter), diff --git a/src/plugins/intel_cpu/src/nodes/llm_mlp.cpp b/src/plugins/intel_cpu/src/nodes/llm_mlp.cpp index 8df1f5498da384..8c598c0641493c 100644 --- a/src/plugins/intel_cpu/src/nodes/llm_mlp.cpp +++ b/src/plugins/intel_cpu/src/nodes/llm_mlp.cpp @@ -501,7 +501,7 @@ struct LLMMLP::Executor : public LLMMLP::ExecutorBase { #endif LLMMLP::LLMMLP(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; const auto & config = context->getConfig(); if (!isSupportedOperation(op, errorMessage, config.fcDynamicQuantizationGroupSize)) { diff --git a/src/plugins/intel_cpu/src/nodes/log_softmax.cpp b/src/plugins/intel_cpu/src/nodes/log_softmax.cpp index 7c0c90ddb2c9e8..8cc1c2bfb8d921 100644 --- a/src/plugins/intel_cpu/src/nodes/log_softmax.cpp +++ b/src/plugins/intel_cpu/src/nodes/log_softmax.cpp @@ -26,7 +26,7 @@ bool LogSoftmax::isSupportedOperation(const std::shared_ptr& op, } LogSoftmax::LogSoftmax(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/memory.cpp b/src/plugins/intel_cpu/src/nodes/memory.cpp index 1c6e2aca6fa4bf..b43085f3e62f96 100644 --- a/src/plugins/intel_cpu/src/nodes/memory.cpp +++ b/src/plugins/intel_cpu/src/nodes/memory.cpp @@ -116,7 +116,7 @@ bool MemoryOutputBase::isSupportedOperation(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) , MemoryNode(op) { + : Node(op, context, NgraphShapeInferFactory(op)), MemoryNode(op) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/mha.cpp b/src/plugins/intel_cpu/src/nodes/mha.cpp index 9364058c5d19a2..97c2d9be4d4ea8 100644 --- a/src/plugins/intel_cpu/src/nodes/mha.cpp +++ b/src/plugins/intel_cpu/src/nodes/mha.cpp @@ -747,7 +747,7 @@ bool MHA::isSupportedOperation(const std::shared_ptr& op, std::s } MHA::MHA(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/multinomial.cpp b/src/plugins/intel_cpu/src/nodes/multinomial.cpp index 24958b4e2b980d..4ea11b2b44ffac 100644 --- a/src/plugins/intel_cpu/src/nodes/multinomial.cpp +++ b/src/plugins/intel_cpu/src/nodes/multinomial.cpp @@ -14,7 +14,7 @@ namespace intel_cpu { namespace node { Multinomial::Multinomial(const std::shared_ptr& op, const GraphContext::CPtr& context) - : Node(op, context, NgraphShapeInferFactory(op, PortMask(NUM_SAMPLES_PORT))) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { THROW_CPU_NODE_ERR(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/mvn.cpp b/src/plugins/intel_cpu/src/nodes/mvn.cpp index 76471b0cca741d..6405a3f1511286 100644 --- a/src/plugins/intel_cpu/src/nodes/mvn.cpp +++ b/src/plugins/intel_cpu/src/nodes/mvn.cpp @@ -1774,7 +1774,7 @@ bool MVN::isSupportedOperation(const std::shared_ptr& op, std::s } MVN::MVN(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/pad.cpp b/src/plugins/intel_cpu/src/nodes/pad.cpp index 10cdb2a19b771f..fb31a0a08fd2a2 100644 --- a/src/plugins/intel_cpu/src/nodes/pad.cpp +++ b/src/plugins/intel_cpu/src/nodes/pad.cpp @@ -42,7 +42,7 @@ bool Pad::isSupportedOperation(const std::shared_ptr& op, std::s } Pad::Pad(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, PortMask(PADS_BEGIN_ID, PADS_END_ID))) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/pooling.cpp b/src/plugins/intel_cpu/src/nodes/pooling.cpp index 71e5c38f0e0a79..c7b687d83c2d25 100644 --- a/src/plugins/intel_cpu/src/nodes/pooling.cpp +++ b/src/plugins/intel_cpu/src/nodes/pooling.cpp @@ -164,7 +164,7 @@ bool Pooling::isSupportedOperation(const std::shared_ptr& op, st } Pooling::Pooling(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/proposal.cpp b/src/plugins/intel_cpu/src/nodes/proposal.cpp index f2cfb448755a9c..cb29e4907de5da 100644 --- a/src/plugins/intel_cpu/src/nodes/proposal.cpp +++ b/src/plugins/intel_cpu/src/nodes/proposal.cpp @@ -92,7 +92,7 @@ bool Proposal::isSupportedOperation(const std::shared_ptr& op, s } Proposal::Proposal(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/psroi_pooling.cpp b/src/plugins/intel_cpu/src/nodes/psroi_pooling.cpp index e0c130e0ba6f63..80b1f577572ce3 100644 --- a/src/plugins/intel_cpu/src/nodes/psroi_pooling.cpp +++ b/src/plugins/intel_cpu/src/nodes/psroi_pooling.cpp @@ -58,7 +58,7 @@ bool PSROIPooling::isSupportedOperation(const std::shared_ptr& o } PSROIPooling::PSROIPooling(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/qkv_proj.cpp b/src/plugins/intel_cpu/src/nodes/qkv_proj.cpp index 00c8b6f9b17c0b..f732fe883f85a1 100644 --- a/src/plugins/intel_cpu/src/nodes/qkv_proj.cpp +++ b/src/plugins/intel_cpu/src/nodes/qkv_proj.cpp @@ -324,7 +324,7 @@ void QKVProjection::execute(dnnl::stream strm) { } QKVProjection::QKVProjection(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; const auto & config = context->getConfig(); diff --git a/src/plugins/intel_cpu/src/nodes/random_uniform.cpp b/src/plugins/intel_cpu/src/nodes/random_uniform.cpp index 808ad10c440854..b40796bf8d93d6 100644 --- a/src/plugins/intel_cpu/src/nodes/random_uniform.cpp +++ b/src/plugins/intel_cpu/src/nodes/random_uniform.cpp @@ -29,7 +29,7 @@ bool RandomUniform::isSupportedOperation(const std::shared_ptr& } RandomUniform::RandomUniform(const std::shared_ptr& op, const GraphContext::CPtr& context) - : Node(op, context, NgraphShapeInferFactory(op, PortMask(0, 1, 2))) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { THROW_CPU_NODE_ERR(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/rdft.cpp b/src/plugins/intel_cpu/src/nodes/rdft.cpp index c7752a3d29b3f6..4e506fb771277f 100644 --- a/src/plugins/intel_cpu/src/nodes/rdft.cpp +++ b/src/plugins/intel_cpu/src/nodes/rdft.cpp @@ -72,8 +72,8 @@ static std::vector getDefaultSignalSizes(const VectorDims& inputShape, cons return signalSizes; } -RDFT::RDFT(const std::shared_ptr& op, const GraphContext::CPtr context) : - Node(op, context, NgraphShapeInferFactory(op, PortMask(1, 2))) { +RDFT::RDFT(const std::shared_ptr& op, const GraphContext::CPtr context) + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/reduce.cpp b/src/plugins/intel_cpu/src/nodes/reduce.cpp index 6cfc94a02b9f3b..1bc0209e0d9c69 100644 --- a/src/plugins/intel_cpu/src/nodes/reduce.cpp +++ b/src/plugins/intel_cpu/src/nodes/reduce.cpp @@ -1895,7 +1895,7 @@ bool Reduce::isSupportedOperation(const std::shared_ptr& op, std } Reduce::Reduce(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, PortMask(REDUCE_INDEXES))) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (isSupportedOperation(op, errorMessage)) { errorPrefix = "Reduce node with name '" + getName() + "'"; @@ -2020,6 +2020,7 @@ void Reduce::initSupportedPrimitiveDescriptors() { config.outConfs[0].setMemDesc(creatorsMap.at(outFormat)->createSharedDesc(outPrecision, getOutputShapeAtPort(0))); if (useAclExecutor) { +#if defined (OV_CPU_WITH_ACL) std::vector srcMemoryDescs; for (size_t i = 0; i < config.inConfs.size(); i++) { srcMemoryDescs.push_back(config.inConfs[i].getMemDesc()); @@ -2034,22 +2035,29 @@ void Reduce::initSupportedPrimitiveDescriptors() { if (!factory->isEmpty()) { supportedPrimitiveDescriptors.push_back({config, impl_type, factory}); } +#endif } else { supportedPrimitiveDescriptors.push_back({config, impl_type}); } }; #if defined (OV_CPU_WITH_ACL) - reduceAttrs.operation = algorithm; - reduceAttrs.keepDims = keep_dims; - reduceAttrs.axes = raw_axes; - for (auto &axis : reduceAttrs.axes) { - if (axis < 0) - axis += static_cast(getInputShapeAtPort(REDUCE_DATA).getRank()); + // acl doesn't support empty input + if (!isDynamicNode() && shape_size(getInputShapeAtPort(REDUCE_DATA).getStaticDims()) == 0) { + canUseAclExecutor = false; + } else { + reduceAttrs.operation = algorithm; + reduceAttrs.keepDims = keep_dims; + reduceAttrs.axes = raw_axes; + for (auto &axis : reduceAttrs.axes) { + if (axis < 0) + axis += static_cast(getInputShapeAtPort(REDUCE_DATA).getRank()); + } + pushDesc(LayoutType::nspc, LayoutType::nspc, input_prec, output_prec, impl_desc_type::undef, true); + pushDesc(LayoutType::ncsp, LayoutType::ncsp, input_prec, output_prec, impl_desc_type::undef, true); + canUseAclExecutor = !supportedPrimitiveDescriptors.empty(); } - pushDesc(LayoutType::nspc, LayoutType::nspc, input_prec, output_prec, impl_desc_type::undef, true); - pushDesc(LayoutType::ncsp, LayoutType::ncsp, input_prec, output_prec, impl_desc_type::undef, true); - canUseAclExecutor = !supportedPrimitiveDescriptors.empty(); + if (canUseAclExecutor) return; #endif @@ -2089,10 +2097,16 @@ void Reduce::initSupportedPrimitiveDescriptors() { } bool Reduce::isExecutable() const { - return !isInputTensorAtPortEmpty(REDUCE_DATA); + return !isOutputTensorAtPortEmpty(0); } void Reduce::prepareParams() { + auto srcMemPtr = getSrcMemoryAtPort(REDUCE_DATA); + auto dstMemPtr = getDstMemoryAtPort(0); + const auto& src_shape = srcMemPtr->getStaticDims(); + dst_size = dstMemPtr->getSize(); + empty_input = shape_size(src_shape) == 0; +#if defined (OV_CPU_WITH_ACL) if (canUseAclExecutor) { std::vector srcMemoryDescs; for (size_t i = 0; i < getParentEdges().size(); i++) { @@ -2102,11 +2116,15 @@ void Reduce::prepareParams() { dstMemoryDescs.push_back(getDstMemoryAtPort(0)->getDescPtr()); auto selectedPD = getSelectedPrimitiveDescriptor(); - aclExecPtr = selectedPD->getExecutorFactoryAs()->makeExecutor(reduceAttrs, srcMemoryDescs, dstMemoryDescs, {}); - selectedPD->setImplementationType(aclExecPtr->getImplType()); - + if (!empty_input) { + aclExecPtr = selectedPD->getExecutorFactoryAs()->makeExecutor(reduceAttrs, srcMemoryDescs, dstMemoryDescs, {}); + selectedPD->setImplementationType(aclExecPtr->getImplType()); + } else { + selectedPD->setImplementationType(acl); + } return; } +#endif src_dims = getParentEdgeAt(REDUCE_DATA)->getMemory().getDesc().getShape().getDims(); std::vector reduce_axes; @@ -2116,9 +2134,7 @@ void Reduce::prepareParams() { reduce_axes = raw_axes; } - auto dstMemPtr = getDstMemoryAtPort(0); const VectorDims &dst_dims = dstMemPtr->getDesc().getShape().getDims(); - dst_size = dstMemPtr->getSize(); calc_process_dst_dims(reduce_axes, dst_dims); if (jit_mode) { set_reduce_dim_flags(); @@ -2274,11 +2290,26 @@ void Reduce::execute(dnnl::stream strm) { const uint8_t *src_data = srcMemPtr->getDataAs(); uint8_t *dst_data = dstMemPtr->getDataAs(); + if (empty_input && dst_size > 0) { +#if defined(OPENVINO_ARCH_X86_64) + output_info_reassign(&dst_data); + init_dst_data(dst_data, dst_size); + output_info_restore(&dst_data); + if (attr.get()->post_ops_.len() != 0) { + reduce_kernel_post_process(dst_data); + } +#else + init_dst_data(dst_data, dst_size); +#endif + return; + } + if (jit_mode) { if (is_hybrid_layout) { dst_data = reinterpret_cast(prc_mem.get_data_handle()); } reduce_type(src_data, dst_data); +#if defined (OV_CPU_WITH_ACL) } else if (aclExecPtr) { std::vector srcMemory; for (size_t i = 0; i < getParentEdges().size(); i++) { @@ -2288,6 +2319,7 @@ void Reduce::execute(dnnl::stream strm) { dstMemory.push_back(getDstMemoryAtPort(0)); aclExecPtr->exec(srcMemory, dstMemory, postOpsDataPtrs.data()); +#endif } else { if (layout == ReduceLayoutType::reduce_ncsp) { auto in_ptr = reinterpret_cast(src_data); @@ -2725,7 +2757,7 @@ inline void Reduce::reduce_kernel_process(const uint8_t *in_p, uint8_t *out_p, s inline void Reduce::reduce_kernel_post_process(uint8_t *out_ptr) { const uint8_t *in_ptr = fuse_low_precision ? static_cast(&intermediate_buf[0]) : nullptr; - const size_t integerDivisor = IB * IC * ID * IH * IW / (OB * OC * OD * OH * OW); + const size_t integerDivisor = empty_input ? 1 : IB * IC * ID * IH * IW / (OB * OC * OD * OH * OW); const float divisor = static_cast(integerDivisor); if (layout == ReduceLayoutType::reduce_ncsp) { parallel_for2d(OB, OC, [&](size_t ob, size_t oc) { diff --git a/src/plugins/intel_cpu/src/nodes/reduce.h b/src/plugins/intel_cpu/src/nodes/reduce.h index 2464686edb1ee4..c790cefb0583b0 100644 --- a/src/plugins/intel_cpu/src/nodes/reduce.h +++ b/src/plugins/intel_cpu/src/nodes/reduce.h @@ -152,6 +152,7 @@ class Reduce : public Node { bool ReduceCDW_opt = false; bool use_aux_kernel = false; bool set_use_aux_kernel = false; + bool empty_input = false; bool ReduceN, ReduceC, ReduceD, ReduceH, ReduceW; size_t IB, IC, ID, IH, IW; size_t OB, OC, OD, OH, OW; @@ -188,9 +189,11 @@ class Reduce : public Node { std::string errorPrefix; +#if defined (OV_CPU_WITH_ACL) ReduceAttrs reduceAttrs; bool canUseAclExecutor = false; std::shared_ptr aclExecPtr = nullptr; +#endif }; } // namespace node diff --git a/src/plugins/intel_cpu/src/nodes/reference.cpp b/src/plugins/intel_cpu/src/nodes/reference.cpp index 185815acd8c294..2c83a4ea45259a 100644 --- a/src/plugins/intel_cpu/src/nodes/reference.cpp +++ b/src/plugins/intel_cpu/src/nodes/reference.cpp @@ -4,14 +4,29 @@ #include "reference.h" #include "common/cpu_memcpy.h" +#include "shape_inference/shape_inference_ngraph.hpp" namespace ov { namespace intel_cpu { + +class ReferenceShapeInferFactory : public ShapeInferFactory { +public: + ReferenceShapeInferFactory(std::shared_ptr op) : m_op{std::move(op)} {} + + ShapeInferPtr makeShapeInfer() const override { + return std::make_shared(make_shape_inference(m_op), FULL_PORT_MASK); + } + +private: + std::shared_ptr m_op; +}; + namespace node { -Reference::Reference(const std::shared_ptr& op, const GraphContext::CPtr& context, - const std::string& errorMessage) : - Node(op, context, NgraphShapeInferFactory(op, FULL_PORT_MASK)), ovCoreNode(op), additionalErrorMessage(errorMessage) { +Reference::Reference(const std::shared_ptr& op, + const GraphContext::CPtr& context, + const std::string& errorMessage) + : Node(op, context, ReferenceShapeInferFactory(op)), ovCoreNode(op), additionalErrorMessage(errorMessage) { if (!op->has_evaluate()) { OPENVINO_THROW_NOT_IMPLEMENTED( "Cannot fallback on ngraph reference implementation (Ngraph::Node::evaluate() is not implemented)"); @@ -70,7 +85,7 @@ void Reference::executeDynamicImpl(dnnl::stream strm) { } } } else { - THROW_CPU_NODE_ERR("got unexpected shape infer result status during the inference."); + THROW_CPU_NODE_ERR("got unexpected shape infer result status during the inference."); } if (!ovCoreNode->evaluate(outputs, inputs)) { THROW_CPU_NODE_ERR("evaluation failed for core operation: ", std::string(ovCoreNode->get_type_name())); @@ -110,9 +125,10 @@ bool Reference::needShapeInfer() const { ov::TensorVector Reference::prepareInputs() const { ov::TensorVector inputs; for (size_t i = 0lu; i < inputShapes.size(); i++) { - void *srcDataPtr = getSrcDataAtPort(i); - ov::Shape shape = ovCoreNode->get_input_partial_shape(i).rank().get_length() == 0 ? - ov::Shape{} : getParentEdgeAt(i)->getMemory().getStaticDims(); + void* srcDataPtr = getSrcDataAtPort(i); + ov::Shape shape = ovCoreNode->get_input_partial_shape(i).rank().get_length() == 0 + ? ov::Shape{} + : getParentEdgeAt(i)->getMemory().getStaticDims(); if (std::any_of(shape.begin(), shape.end(), [](const size_t dim) { return dim == 0lu; } )) { inputs.push_back(ov::Tensor(ovCoreNode->get_input_element_type(i), shape)); @@ -127,9 +143,10 @@ ov::TensorVector Reference::prepareInputs() const { ov::TensorVector Reference::prepareOutputs() const { ov::TensorVector outputs; for (size_t i = 0lu; i < outputShapes.size(); i++) { - void *dstDataPtr = getDstDataAtPort(i); - ov::Shape shape = ovCoreNode->get_output_partial_shape(i).rank().get_length() == 0 ? - ov::Shape{} : getChildEdgeAt(i)->getMemory().getStaticDims(); + void* dstDataPtr = getDstDataAtPort(i); + ov::Shape shape = ovCoreNode->get_output_partial_shape(i).rank().get_length() == 0 + ? ov::Shape{} + : getChildEdgeAt(i)->getMemory().getStaticDims(); if (std::any_of(shape.begin(), shape.end(), [](const size_t dim) { return dim == 0lu; } )) { outputs.push_back(ov::Tensor(ovCoreNode->get_output_element_type(i), shape)); diff --git a/src/plugins/intel_cpu/src/nodes/region_yolo.cpp b/src/plugins/intel_cpu/src/nodes/region_yolo.cpp index 8ea69feea303f9..c575237f48e042 100644 --- a/src/plugins/intel_cpu/src/nodes/region_yolo.cpp +++ b/src/plugins/intel_cpu/src/nodes/region_yolo.cpp @@ -247,7 +247,7 @@ bool RegionYolo::needPrepareParams() const { } RegionYolo::RegionYolo(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/reorg_yolo.cpp b/src/plugins/intel_cpu/src/nodes/reorg_yolo.cpp index 1f37e15edae1ff..49ad9b9cec5777 100644 --- a/src/plugins/intel_cpu/src/nodes/reorg_yolo.cpp +++ b/src/plugins/intel_cpu/src/nodes/reorg_yolo.cpp @@ -26,7 +26,7 @@ bool ReorgYolo::isSupportedOperation(const std::shared_ptr& op, } ReorgYolo::ReorgYolo(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/reverse_sequence.cpp b/src/plugins/intel_cpu/src/nodes/reverse_sequence.cpp index b51eab4bef393e..66139415588a10 100644 --- a/src/plugins/intel_cpu/src/nodes/reverse_sequence.cpp +++ b/src/plugins/intel_cpu/src/nodes/reverse_sequence.cpp @@ -27,7 +27,7 @@ bool ReverseSequence::isSupportedOperation(const std::shared_ptr } ReverseSequence::ReverseSequence(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/roi_align.cpp b/src/plugins/intel_cpu/src/nodes/roi_align.cpp index 27f9426dca6af9..2190ebd9445df8 100644 --- a/src/plugins/intel_cpu/src/nodes/roi_align.cpp +++ b/src/plugins/intel_cpu/src/nodes/roi_align.cpp @@ -674,7 +674,7 @@ bool ROIAlign::isSupportedOperation(const std::shared_ptr& op, s } ROIAlign::ROIAlign(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (isSupportedOperation(op, errorMessage)) { errorPrefix = "ROIPooling layer with name '" + getName() + "' "; diff --git a/src/plugins/intel_cpu/src/nodes/roi_align_rotated.cpp b/src/plugins/intel_cpu/src/nodes/roi_align_rotated.cpp index e75bef1213d2cf..fbe8127b9f247a 100644 --- a/src/plugins/intel_cpu/src/nodes/roi_align_rotated.cpp +++ b/src/plugins/intel_cpu/src/nodes/roi_align_rotated.cpp @@ -14,7 +14,7 @@ namespace intel_cpu { namespace node { ROIAlignRotated::ROIAlignRotated(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { const auto roiAlign = ov::as_type_ptr(op); pooledH = roiAlign->get_pooled_h(); pooledW = roiAlign->get_pooled_w(); diff --git a/src/plugins/intel_cpu/src/nodes/roi_pooling.cpp b/src/plugins/intel_cpu/src/nodes/roi_pooling.cpp index 7458457c7b540e..816adb3ffa0666 100644 --- a/src/plugins/intel_cpu/src/nodes/roi_pooling.cpp +++ b/src/plugins/intel_cpu/src/nodes/roi_pooling.cpp @@ -388,7 +388,7 @@ bool ROIPooling::isSupportedOperation(const std::shared_ptr& op, } ROIPooling::ROIPooling(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/roll.cpp b/src/plugins/intel_cpu/src/nodes/roll.cpp index 6f75361c13c37f..7b17dff9746666 100644 --- a/src/plugins/intel_cpu/src/nodes/roll.cpp +++ b/src/plugins/intel_cpu/src/nodes/roll.cpp @@ -33,8 +33,8 @@ bool Roll::isSupportedOperation(const std::shared_ptr& op, std:: return true; } -Roll::Roll(const std::shared_ptr& op, const GraphContext::CPtr context) : - Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { +Roll::Roll(const std::shared_ptr& op, const GraphContext::CPtr context) + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (isSupportedOperation(op, errorMessage)) { layerErrorPrefix = "Roll layer with name '" + getName() + "'"; diff --git a/src/plugins/intel_cpu/src/nodes/rope.cpp b/src/plugins/intel_cpu/src/nodes/rope.cpp index f089b67a122beb..9144eaaae846bd 100644 --- a/src/plugins/intel_cpu/src/nodes/rope.cpp +++ b/src/plugins/intel_cpu/src/nodes/rope.cpp @@ -22,7 +22,7 @@ namespace intel_cpu { namespace node { RoPE::RoPE(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW("CPU: " + errorMessage); @@ -392,7 +392,7 @@ void RoPE::initSupportedPrimitiveDescriptors() { m_executor = std::make_shared>(m_config); rtPrecision = ov::element::f32; } - } else if (m_config.is_interleaved) { + } else if (m_config.is_interleaved && m_config.output_trans0213) { OPENVINO_ASSERT(m_config.input_trans0213 == false); OPENVINO_ASSERT(m_config.slice_start == 0); OPENVINO_ASSERT(m_config.slice_stop == 0); diff --git a/src/plugins/intel_cpu/src/nodes/scatter_update.cpp b/src/plugins/intel_cpu/src/nodes/scatter_update.cpp index 76c9acd218d9d1..3934f4f5bc0120 100644 --- a/src/plugins/intel_cpu/src/nodes/scatter_update.cpp +++ b/src/plugins/intel_cpu/src/nodes/scatter_update.cpp @@ -58,11 +58,13 @@ bool ScatterUpdate::isExecutable() const { } ScatterUpdate::ScatterUpdate(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)), - dataSize(0lu), indicesSize(0lu), axisSize(0lu), - dataPrec(ov::element::undefined), - indicesPrec(ov::element::undefined), - axisPrec(ov::element::undefined) { + : Node(op, context, NgraphShapeInferFactory(op)), + dataSize(0lu), + indicesSize(0lu), + axisSize(0lu), + dataPrec(ov::element::undefined), + indicesPrec(ov::element::undefined), + axisPrec(ov::element::undefined) { std::string errorMessage; if (isSupportedOperation(op, errorMessage)) { errorPrefix = std::string(op->get_type_name()) + " node with name '" + getName() + "'"; @@ -979,6 +981,7 @@ void ScatterUpdate::scatterNDUpdate(const MemoryPtr& mem_data, uint8_t* update = mem_updates->getDataAs(); uint8_t* dstData = mem_data->getDataAs(); const auto& srcDataDim = getParentEdgeAt(DATA_ID)->getMemory().getStaticDims(); + const auto elementsCount = getParentEdgeAt(DATA_ID)->getMemory().getShape().getElementsCount(); const auto& indicesDim = getParentEdgeAt(INDICES_ID)->getMemory().getStaticDims(); size_t indicesRank = indicesDim.size(); @@ -1002,8 +1005,14 @@ void ScatterUpdate::scatterNDUpdate(const MemoryPtr& mem_data, } dstOffset += idxValue * srcBlockND[i + 1]; } + + // Exception must be thrown according to the specification + CPU_NODE_ASSERT(dstOffset < elementsCount, + " indices contain values that points to non-existing data tensor element"); + dstOffset *= dataSize; size_t updateOffset = tupleIdx * sizeToUpdate; + cpu_memcpy(dstData + dstOffset, update + updateOffset, sizeToUpdate); }); } @@ -1018,6 +1027,7 @@ void ScatterUpdate::scatterNDUpdate(const MemoryPtr& mem_data, DataType* update = mem_updates->getDataAs(); DataType* dstData = mem_data->getDataAs(); const auto& srcDataDim = getParentEdgeAt(DATA_ID)->getMemory().getStaticDims(); + const auto elementsCount = getParentEdgeAt(DATA_ID)->getMemory().getShape().getElementsCount(); const auto& indicesDim = getParentEdgeAt(INDICES_ID)->getMemory().getStaticDims(); const auto indicesRank = indicesDim.size(); @@ -1039,6 +1049,11 @@ void ScatterUpdate::scatterNDUpdate(const MemoryPtr& mem_data, } dstOffset += idxValue * srcBlockND[i + 1]; } + + // Exception must be thrown according to the specification + CPU_NODE_ASSERT(dstOffset < elementsCount, + " indices contain values that points to non-existing data tensor element"); + const auto updateOffset = tupleIdx * sizeToUpdate; DataType* dstDataWithOffset = dstData + dstOffset; const DataType* updateWithOffset = update + updateOffset; diff --git a/src/plugins/intel_cpu/src/nodes/search_sorted.cpp b/src/plugins/intel_cpu/src/nodes/search_sorted.cpp index e88177659167d6..69e57398bd327d 100644 --- a/src/plugins/intel_cpu/src/nodes/search_sorted.cpp +++ b/src/plugins/intel_cpu/src/nodes/search_sorted.cpp @@ -11,7 +11,7 @@ namespace ov { namespace intel_cpu { namespace node { SearchSorted::SearchSorted(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/shuffle_channels.cpp b/src/plugins/intel_cpu/src/nodes/shuffle_channels.cpp index dd45a639e98847..974001a78c994a 100644 --- a/src/plugins/intel_cpu/src/nodes/shuffle_channels.cpp +++ b/src/plugins/intel_cpu/src/nodes/shuffle_channels.cpp @@ -67,7 +67,7 @@ bool ShuffleChannels::isSupportedOperation(const std::shared_ptr } ShuffleChannels::ShuffleChannels(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/space_to_batch.cpp b/src/plugins/intel_cpu/src/nodes/space_to_batch.cpp index f3da2510241d67..044200ba183e4f 100644 --- a/src/plugins/intel_cpu/src/nodes/space_to_batch.cpp +++ b/src/plugins/intel_cpu/src/nodes/space_to_batch.cpp @@ -25,7 +25,7 @@ bool SpaceToBatch::isSupportedOperation(const std::shared_ptr& o } SpaceToBatch::SpaceToBatch(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, PortMask(1, 2, 3))) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/space_to_depth.cpp b/src/plugins/intel_cpu/src/nodes/space_to_depth.cpp index 6753a9510bdc2f..0fcb190035537c 100644 --- a/src/plugins/intel_cpu/src/nodes/space_to_depth.cpp +++ b/src/plugins/intel_cpu/src/nodes/space_to_depth.cpp @@ -73,7 +73,7 @@ bool SpaceToDepth::isSupportedOperation(const std::shared_ptr& o } SpaceToDepth::SpaceToDepth(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, EMPTY_PORT_MASK)) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/split.cpp b/src/plugins/intel_cpu/src/nodes/split.cpp index 72af54e619dbf3..aca963536d8059 100644 --- a/src/plugins/intel_cpu/src/nodes/split.cpp +++ b/src/plugins/intel_cpu/src/nodes/split.cpp @@ -45,8 +45,8 @@ bool Split::isSupportedOperation(const std::shared_ptr& op, std: return true; } -Split::Split(const std::shared_ptr& op, const GraphContext::CPtr context) : - Node(op, context, NgraphShapeInferFactory(op, PortMask(1, 2))) { +Split::Split(const std::shared_ptr& op, const GraphContext::CPtr context) + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/stft.cpp b/src/plugins/intel_cpu/src/nodes/stft.cpp index d03bded6ad5e43..47855a7eff7399 100644 --- a/src/plugins/intel_cpu/src/nodes/stft.cpp +++ b/src/plugins/intel_cpu/src/nodes/stft.cpp @@ -26,7 +26,7 @@ bool STFT::isSupportedOperation(const std::shared_ptr& op, std:: } STFT::STFT(const std::shared_ptr& op, const GraphContext::CPtr& context) - : Node(op, context, NgraphShapeInferFactory(op, PortMask(2, 3))) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { THROW_CPU_NODE_ERR(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/string_tensor_pack.cpp b/src/plugins/intel_cpu/src/nodes/string_tensor_pack.cpp index 5ecb15b57fbb71..f8d6c2c78fecf2 100644 --- a/src/plugins/intel_cpu/src/nodes/string_tensor_pack.cpp +++ b/src/plugins/intel_cpu/src/nodes/string_tensor_pack.cpp @@ -10,7 +10,7 @@ namespace ov { namespace intel_cpu { namespace node { StringTensorPack::StringTensorPack(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, PortMask(0, 1))) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/tile.cpp b/src/plugins/intel_cpu/src/nodes/tile.cpp index 447898e6390342..1d197a9c1d6ca0 100644 --- a/src/plugins/intel_cpu/src/nodes/tile.cpp +++ b/src/plugins/intel_cpu/src/nodes/tile.cpp @@ -35,8 +35,8 @@ bool Tile::isSupportedOperation(const std::shared_ptr& op, std:: return true; } -Tile::Tile(const std::shared_ptr& op, const GraphContext::CPtr context) : - Node(op, context, NgraphShapeInferFactory(op, PortMask(TILE_REPEATS))) { +Tile::Tile(const std::shared_ptr& op, const GraphContext::CPtr context) + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); diff --git a/src/plugins/intel_cpu/src/nodes/topk.cpp b/src/plugins/intel_cpu/src/nodes/topk.cpp index 04602d5bfe69a6..1210fbe5ae8bfd 100644 --- a/src/plugins/intel_cpu/src/nodes/topk.cpp +++ b/src/plugins/intel_cpu/src/nodes/topk.cpp @@ -1823,7 +1823,7 @@ bool TopK::isSupportedOperation(const std::shared_ptr& op, std:: } TopK::TopK(const std::shared_ptr& op, const GraphContext::CPtr context) - : Node(op, context, NgraphShapeInferFactory(op, PortMask(TOPK_K))) { + : Node(op, context, NgraphShapeInferFactory(op)) { std::string errorMessage; if (isSupportedOperation(op, errorMessage)) { errorPrefix = "TopK layer with name '" + getName() + "'"; diff --git a/src/plugins/intel_cpu/src/shape_inference/shape_inference.cpp b/src/plugins/intel_cpu/src/shape_inference/shape_inference.cpp index 8e2e794eacc939..b99e1bc62c4b11 100644 --- a/src/plugins/intel_cpu/src/shape_inference/shape_inference.cpp +++ b/src/plugins/intel_cpu/src/shape_inference/shape_inference.cpp @@ -214,13 +214,12 @@ class ShapeInferFallback : public ShapeInferBase { ov::OutputVector new_inputs; for (size_t i = 0; i < op->get_input_size(); ++i) { if (auto t = tensor_accessor(i)) { - new_inputs.push_back( - std::make_shared(t)); - } else if (dynamic_cast(op->get_input_node_ptr(i))) { - new_inputs.push_back(op->get_input_node_ptr(i)->clone_with_new_inputs(ov::OutputVector{})); + new_inputs.push_back(std::make_shared(t)); + } else if (auto c = ov::as_type(op->get_input_node_ptr(i))) { + new_inputs.push_back(c->clone_with_new_inputs(ov::OutputVector{})); } else { - new_inputs.push_back(std::make_shared(op->get_input_element_type(i), - input_shapes[i].to_partial_shape())); + new_inputs.push_back(std::make_shared(op->get_input_element_type(i), + input_shapes[i].to_partial_shape())); } } local_op = op->clone_with_new_inputs(new_inputs); @@ -461,7 +460,6 @@ const IStaticShapeInferFactory::TRegistry IStaticShapeInferFactory::registry{ _OV_OP_SHAPE_INFER_VA_REG(opset7::Gather, ShapeInferTA, op::util::GatherBase, util::bit::mask(2)), // opset6 _OV_OP_SHAPE_INFER_MASK_REG(opset6::CTCGreedyDecoderSeqLen, ShapeInferTA, util::bit::mask()), - _OV_OP_SHAPE_INFER_MASK_REG(opset6::CTCGreedyDecoderSeqLen, ShapeInferTA, util::bit::mask()), _OV_OP_SHAPE_INFER_MASK_REG(opset6::ExperimentalDetectronDetectionOutput, ShapeInferTA, util::bit::mask()), _OV_OP_SHAPE_INFER_MASK_REG(opset6::ExperimentalDetectronGenerateProposalsSingleImage, ShapeInferTA, util::bit::mask()), _OV_OP_SHAPE_INFER_MASK_REG(opset6::ExperimentalDetectronPriorGridGenerator, ShapeInferTA, util::bit::mask()), @@ -565,9 +563,6 @@ const IStaticShapeInferFactory::TRegistry IStaticShapeInferFactory::registry{ _OV_OP_SHAPE_INFER_MASK_REG(opset1::Unsqueeze, ShapeInferTA, util::bit::mask(1)), _OV_OP_SHAPE_INFER_MASK_REG(opset1::VariadicSplit, ShapeInferTA, util::bit::mask(1, 2)), _OV_OP_SHAPE_INFER_NON_TEMPLATE_REG(opset1::BatchNormInference, ShapeInferBase), - _OV_OP_SHAPE_INFER_NON_TEMPLATE_REG(opset1::BatchNormInference, ShapeInferBase), - _OV_OP_SHAPE_INFER_NON_TEMPLATE_REG(opset1::Convert, ShapeInferCopy), - _OV_OP_SHAPE_INFER_NON_TEMPLATE_REG(opset1::Convert, ShapeInferCopy), _OV_OP_SHAPE_INFER_NON_TEMPLATE_REG(opset1::Convert, ShapeInferCopy), _OV_OP_SHAPE_INFER_NON_TEMPLATE_REG(opset1::HardSigmoid, ShapeInferBase), _OV_OP_SHAPE_INFER_NON_TEMPLATE_REG(opset1::LogicalNot, ShapeInferCopy), diff --git a/src/plugins/intel_cpu/src/shape_inference/shape_inference_cpu.cpp b/src/plugins/intel_cpu/src/shape_inference/shape_inference_cpu.cpp index 2c6893ad1a38d0..3b759c21803092 100644 --- a/src/plugins/intel_cpu/src/shape_inference/shape_inference_cpu.cpp +++ b/src/plugins/intel_cpu/src/shape_inference/shape_inference_cpu.cpp @@ -4,10 +4,15 @@ #include "shape_inference_ngraph.hpp" -using namespace ov::intel_cpu; +namespace ov { +namespace intel_cpu { +NgraphShapeInferFactory::NgraphShapeInferFactory(std::shared_ptr op) : m_op(std::move(op)) {} ShapeInferPtr NgraphShapeInferFactory::makeShapeInfer() const { - return std::make_shared(make_shape_inference(m_op), m_port_mask); + return std::make_shared(make_shape_inference(m_op)); } const ov::CoordinateDiff ShapeInferEmptyPads::m_emptyVec = {}; + +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/shape_inference/shape_inference_cpu.hpp b/src/plugins/intel_cpu/src/shape_inference/shape_inference_cpu.hpp index 99af2bac163e24..661db2a739b57c 100644 --- a/src/plugins/intel_cpu/src/shape_inference/shape_inference_cpu.hpp +++ b/src/plugins/intel_cpu/src/shape_inference/shape_inference_cpu.hpp @@ -5,8 +5,9 @@ #pragma once #include -#include -#include + +#include "openvino/core/coordinate_diff.hpp" +#include "openvino/core/node.hpp" #include "shape_inference_status.hpp" namespace ov { @@ -98,14 +99,13 @@ class NgraphShapeInferFactory final : public ShapeInferFactory { * @brief Construct a new Ngraph Shape Infer Factory object * * @param op ngraph operation - * @param port_mask port mask should be defined by the user. Will be stored in the shape infer object and returned - * by the get_port_mask() call */ - NgraphShapeInferFactory(std::shared_ptr op, IShapeInfer::port_mask_t port_mask) : m_op(op), m_port_mask(port_mask) {} + NgraphShapeInferFactory(std::shared_ptr op); + ShapeInferPtr makeShapeInfer() const override; + private: std::shared_ptr m_op; - IShapeInfer::port_mask_t m_port_mask; }; } // namespace intel_cpu } // namespace ov diff --git a/src/plugins/intel_cpu/src/shape_inference/shape_inference_ngraph.cpp b/src/plugins/intel_cpu/src/shape_inference/shape_inference_ngraph.cpp index 3a4dcbdccce096..897e28aad42c78 100644 --- a/src/plugins/intel_cpu/src/shape_inference/shape_inference_ngraph.cpp +++ b/src/plugins/intel_cpu/src/shape_inference/shape_inference_ngraph.cpp @@ -9,6 +9,14 @@ using namespace ov::intel_cpu; +NgraphShapeInfer::NgraphShapeInfer(std::shared_ptr shape_infer) + : m_shape_infer(std::move(shape_infer)), + m_port_mask(m_shape_infer->get_port_mask()) {} + +NgraphShapeInfer::NgraphShapeInfer(std::shared_ptr shape_infer, IShapeInfer::port_mask_t port_mask) + : m_shape_infer(std::move(shape_infer)), + m_port_mask(port_mask) {} + IShapeInfer::Result NgraphShapeInfer::infer( const std::vector>& input_shapes, diff --git a/src/plugins/intel_cpu/src/shape_inference/shape_inference_ngraph.hpp b/src/plugins/intel_cpu/src/shape_inference/shape_inference_ngraph.hpp index 01ba907b991af9..450d4e94055946 100644 --- a/src/plugins/intel_cpu/src/shape_inference/shape_inference_ngraph.hpp +++ b/src/plugins/intel_cpu/src/shape_inference/shape_inference_ngraph.hpp @@ -11,14 +11,24 @@ namespace ov { namespace intel_cpu { /** - * This class wraps core specific shape inference class to implement CPU plugin specific interface. - * + * @brief This class wraps core specific shape inference class to implement CPU plugin specific interface. */ class NgraphShapeInfer : public IShapeInfer { public: - NgraphShapeInfer(std::shared_ptr shape_infer, IShapeInfer::port_mask_t port_mask) - : m_shape_infer(shape_infer), - m_port_mask(port_mask) {} + /** + * @brief Wraps IStaticShapeInfer into IShapeInfer interface. Will use port mask defined in IStaticShapeInfer. + * + * @param shape_infer Instance of IStaticShapeInfer. + */ + NgraphShapeInfer(std::shared_ptr shape_infer); + + /** + * @brief Wraps IStaticShapeInfer into IShapeInfer interface. Will use port mask defined by user + * + * @param shape_infer Instance of IStaticShapeInfer. + * @param port_mask Port mask define. + */ + NgraphShapeInfer(std::shared_ptr shape_infer, IShapeInfer::port_mask_t port_mask); Result infer( const std::vector>& input_shapes, @@ -34,6 +44,7 @@ class NgraphShapeInfer : public IShapeInfer { port_mask_t get_port_mask() const override { return m_port_mask; } + private: std::shared_ptr m_shape_infer; IShapeInfer::port_mask_t m_port_mask; diff --git a/src/plugins/intel_cpu/src/transformations/cpu_opset/arm/pass/convert_reduce_multi_axis.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/arm/pass/convert_reduce_multi_axis.cpp index 22692e644e6e56..ff5632cb0a5e8f 100644 --- a/src/plugins/intel_cpu/src/transformations/cpu_opset/arm/pass/convert_reduce_multi_axis.cpp +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/arm/pass/convert_reduce_multi_axis.cpp @@ -23,6 +23,9 @@ ov::matcher_pass_callback ov::intel_cpu::ConvertReduceMultiAxisBase::convert_red if (!reduction_axes) { return false; } + if (!reduce->is_dynamic() && ov::shape_size(input0.get_shape()) == 0) { + return false; + } if (ov::shape_size(input1.get_shape()) <= 1) { return false; } diff --git a/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp b/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp index 8daf8d81704301..c2ee419c1a1bab 100644 --- a/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp +++ b/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp @@ -846,6 +846,7 @@ void Transformations::PostLpt() { CPU_REGISTER_PASS_X64(postLPTPassManager, ov::pass::RoPEFusion, true); CPU_REGISTER_PASS_ARM64(postLPTPassManager, ov::pass::RoPEFusion, true); + CPU_DISABLE_PASS_COMMON(postLPTPassManager, ov::pass::RoPEFusionFlux); CPU_REGISTER_PASS_X64(postLPTPassManager, CausalMaskPreprocessFusion); #if defined(OPENVINO_ARCH_X86_64) diff --git a/src/plugins/intel_cpu/src/utils/debug_caps_config.cpp b/src/plugins/intel_cpu/src/utils/debug_caps_config.cpp index cef5e5e0e19721..335137a2a0afc5 100644 --- a/src/plugins/intel_cpu/src/utils/debug_caps_config.cpp +++ b/src/plugins/intel_cpu/src/utils/debug_caps_config.cpp @@ -54,15 +54,18 @@ void DebugCapsConfig::readProperties() { if ((envVarValue = readEnv("OV_CPU_BLOB_DUMP_NODE_NAME"))) blobDumpFilters[FILTER::BY_NAME] = envVarValue; - if ((envVarValue = readEnv("OV_CPU_SUMMARY_PERF"))) { - summaryPerf = envVarValue; - } - if ((envVarValue = readEnv("OV_CPU_DISABLE"))) disable.parseAndSet(envVarValue); if ((envVarValue = readEnv("OV_CPU_DUMP_IR"))) dumpIR.parseAndSet(envVarValue); + + if ((envVarValue = readEnv("OV_CPU_SUMMARY_PERF"))) { + summaryPerf = envVarValue; + } + + if ((envVarValue = readEnv("OV_CPU_AVERAGE_COUNTERS"))) + averageCountersPath = envVarValue; } } // namespace intel_cpu diff --git a/src/plugins/intel_cpu/src/utils/debug_caps_config.h b/src/plugins/intel_cpu/src/utils/debug_caps_config.h index 9472b46441e800..096d9b231d07fa 100644 --- a/src/plugins/intel_cpu/src/utils/debug_caps_config.h +++ b/src/plugins/intel_cpu/src/utils/debug_caps_config.h @@ -40,6 +40,7 @@ class DebugCapsConfig { }; std::string execGraphPath; + std::string averageCountersPath; std::string verbose; std::string blobDumpDir = "cpu_dump"; FORMAT blobDumpFormat = FORMAT::TEXT; diff --git a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/eltwise.cpp b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/eltwise.cpp index d7cfe80d22f617..1696f35fc1bc4a 100644 --- a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/eltwise.cpp +++ b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/eltwise.cpp @@ -259,6 +259,7 @@ std::string EltwiseLayerCPUTest::getPrimitiveType(const utils::EltwiseTypes& elt (eltwise_type == utils::EltwiseTypes::MULTIPLY) || (eltwise_type == utils::EltwiseTypes::SUBTRACT) || (eltwise_type == utils::EltwiseTypes::DIVIDE) || + (eltwise_type == utils::EltwiseTypes::FLOOR_MOD) || (eltwise_type == utils::EltwiseTypes::MOD)) { return "jit"; } @@ -317,6 +318,8 @@ const std::vector& eltwiseOpTypesBinInp() { utils::EltwiseTypes::SUBTRACT, // TODO: Fix CVS-105430 utils::EltwiseTypes::DIVIDE, // TODO: Fix CVS-105430 utils::EltwiseTypes::FLOOR_MOD, // TODO: Fix CVS-111875 +#elif defined(OPENVINO_ARCH_ARM64) + utils::EltwiseTypes::FLOOR_MOD, #endif utils::EltwiseTypes::SQUARED_DIFF, utils::EltwiseTypes::MOD, diff --git a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/reduce.cpp b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/reduce.cpp index 66b1d60932b262..b379655338aaf9 100644 --- a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/reduce.cpp +++ b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/reduce.cpp @@ -243,6 +243,25 @@ const std::vector& reductionTypes() { return reductionTypes; } +const std::vector& reductionTypesArithmetic() { + static const std::vector reductionTypesArithmetic = { + ov::test::utils::ReductionType::Mean, + ov::test::utils::ReductionType::Sum, + ov::test::utils::ReductionType::Prod, + ov::test::utils::ReductionType::L1, + ov::test::utils::ReductionType::L2, + }; + return reductionTypesArithmetic; +} + +const std::vector& reductionTypesCompare() { + static const std::vector reductionTypesCompare = { + ov::test::utils::ReductionType::Max, + ov::test::utils::ReductionType::Min, + }; + return reductionTypesCompare; +} + const std::vector& inpOutPrc() { static const std::vector inpOutPrc = {ElementType::f32}; return inpOutPrc; diff --git a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/reduce.hpp b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/reduce.hpp index b3e5fff2ba0b7b..d45cce4b3f913e 100644 --- a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/reduce.hpp +++ b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/classes/reduce.hpp @@ -52,6 +52,8 @@ const std::vector>& axes(); const std::vector>& axesND(); const std::vector& opTypes(); const std::vector& reductionTypes(); +const std::vector& reductionTypesArithmetic(); +const std::vector& reductionTypesCompare(); const std::vector& inpOutPrc(); const std::vector> additionalConfig(); const std::vector> additionalConfigFP32(); diff --git a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/instances/arm/reduce.cpp b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/instances/arm/reduce.cpp index 6d1aa855c31865..45ecc774b5dbf9 100644 --- a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/instances/arm/reduce.cpp +++ b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/instances/arm/reduce.cpp @@ -20,6 +20,11 @@ std::vector> inputShapes_5D = { {{{}, {{2, 19, 2, 2, 9}}}}, }; +std::vector> inputShapes_5D_ZeroDim = { + {{{}, {{2, 19, 0, 2, 9}}}}, + {{{}, {{2, 19, 0, 2, 0}}}}, +}; + const std::vector> axes5D = { {2, 4}, {1, 2, 4}, @@ -70,6 +75,20 @@ const auto params_MultiAxis_5D_ref = testing::Combine( testing::Values(emptyFusingSpec), testing::ValuesIn(config_infer_prec_f32)); +const auto params_MultiAxis_5D_ZeroDim_ref = testing::Combine( + testing::Combine( + testing::ValuesIn(axes5D), + testing::Values(ov::test::utils::OpType::VECTOR), + testing::Values(true), + testing::ValuesIn(reductionTypes()), + testing::ValuesIn(inpOutPrc()), + testing::Values(ElementType::undefined), + testing::Values(ElementType::undefined), + testing::ValuesIn(inputShapes_5D_ZeroDim)), + testing::ValuesIn(filterCPUSpecificParams(cpuParams_5D_ref)), + testing::Values(emptyFusingSpec), + testing::ValuesIn(additionalConfigFP32())); + //There are dedicated instences of smoke_Reduce_MultiAxis_5D_CPU test in arm and x64 folders //because ACL does not support 0 as reduction axis INSTANTIATE_TEST_SUITE_P( @@ -87,6 +106,13 @@ INSTANTIATE_TEST_SUITE_P( ReduceCPULayerTest::getTestCaseName ); +INSTANTIATE_TEST_SUITE_P( + smoke_Reduce_MultiAxis_5D_ZeroDim_CPU_ref, + ReduceCPULayerTest, + params_MultiAxis_5D_ZeroDim_ref, + ReduceCPULayerTest::getTestCaseName +); + } // namespace } // namespace Reduce } // namespace test diff --git a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/instances/x64/reduce.cpp b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/instances/x64/reduce.cpp index 302e47fd45aa84..af4319d66a6efe 100644 --- a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/instances/x64/reduce.cpp +++ b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/instances/x64/reduce.cpp @@ -53,6 +53,12 @@ std::vector> inputShapes_SingleBatch_dyn = { {{{{1, 5}, 19, {1, 5}, {1, 10}}, {{1, 19, 2, 2}, {1, 19, 2, 9}}}}, }; +std::vector> inputShapes_Dynmic_ZeroDim = { + {{{-1, -1, -1, -1}, {{2, 0, 3, 9}}}}, + {{{2, 0, -1, -1}, {{2, 0, 3, 9}}}}, + {{{2, 0, -1, -1}, {{2, 0, 3, 0}}}} +}; + std::vector cpuParams_3D = { CPUSpecificParams({ncw}, {ncw}, {}, {}), }; @@ -99,6 +105,10 @@ const std::vector> axesGather = { {3} }; +const std::vector> axesZeroDimFusing = { + {1, 3}, +}; + std::vector cpuParams_5D = { CPUSpecificParams({nCdhw16c}, {nCdhw16c}, {}, {}), CPUSpecificParams({ndhwc}, {ndhwc}, {}, {}), @@ -144,6 +154,17 @@ const auto fusingFakeQuantizeTranspose = fusingSpecificParams{std::make_shared

fusingParamsFullSet { + emptyFusingSpec, + /* activations */ + fusingSwish, + /* FQ */ + fusingFakeQuantizePerChannelRelu, + fusingFakeQuantizePerTensorRelu, + /* another patterns */ + fusingScaleShift +}; + const std::vector fusingParamsSet { /* activations */ fusingSwish, @@ -600,6 +621,34 @@ const auto params_LowPrecision_fusing = testing::Combine( testing::ValuesIn(fusingParamsSet_LowPrecision), testing::ValuesIn(additionalConfig())); +const auto params_DimZero_Arithmetic_fusing = testing::Combine( + testing::Combine( + testing::ValuesIn(axesZeroDimFusing), + testing::Values(ov::test::utils::OpType::VECTOR), + testing::Values(true), + testing::ValuesIn(reductionTypesArithmetic()), + testing::ValuesIn(inpOutPrc()), + testing::Values(ElementType::undefined), + testing::Values(ElementType::undefined), + testing::ValuesIn(inputShapes_Dynmic_ZeroDim)), + testing::Values(emptyCPUSpec), + testing::ValuesIn(fusingParamsFullSet), + testing::ValuesIn(additionalConfig())); + +const auto params_DimZero_Compare_fusing = testing::Combine( + testing::Combine( + testing::ValuesIn(axesZeroDimFusing), + testing::Values(ov::test::utils::OpType::VECTOR), + testing::Values(true), + testing::ValuesIn(reductionTypesCompare()), + testing::ValuesIn(inpOutPrc()), + testing::Values(ElementType::undefined), + testing::Values(ElementType::undefined), + testing::ValuesIn(inputShapes_Dynmic_ZeroDim)), + testing::Values(emptyCPUSpec), + testing::ValuesIn(fusingParamsFullSet), + testing::ValuesIn(additionalConfigFP32())); + INSTANTIATE_TEST_SUITE_P( smoke_Reduce_OneAxis_fusing_CPU, ReduceCPULayerTest, @@ -635,6 +684,20 @@ INSTANTIATE_TEST_SUITE_P( ReduceCPULayerTest::getTestCaseName ); +INSTANTIATE_TEST_SUITE_P( + smoke_Reduce_DimZero_Arithmetic_fusing_CPU, + ReduceCPULayerTest, + params_DimZero_Arithmetic_fusing, + ReduceCPULayerTest::getTestCaseName +); + +INSTANTIATE_TEST_SUITE_P( + smoke_Reduce_DimZero_Compare_fusing_CPU, + ReduceCPULayerTest, + params_DimZero_Compare_fusing, + ReduceCPULayerTest::getTestCaseName +); + /* ================================ 2.2 Fusion - KeepNoDims ================================ */ const auto params_OneAxis_fusing_KeepNoDims = testing::Combine( testing::Combine( @@ -702,4 +765,4 @@ INSTANTIATE_TEST_SUITE_P( } // namespace } // namespace Reduce } // namespace test -} // namespace ov \ No newline at end of file +} // namespace ov diff --git a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/scatter_ND_update.cpp b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/scatter_ND_update.cpp index 4ffa9a58486344..27f0540dc7dbd7 100644 --- a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/scatter_ND_update.cpp +++ b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/scatter_ND_update.cpp @@ -15,6 +15,7 @@ using IndicesValues = std::vector; struct ScatterNDUpdateLayerParams { ScatterNDUpdateShapes inputShapes; IndicesValues indicesValues; + bool exceptionExpected; }; using scatterUpdateParams = std::tuple scatterParams = { {{2, 2, 1}, {{2, 2, 1}, {2, 2, 1}, {2, 2, 1}}}, {{-1, -1, -1, -1, -1, -1}, {{2, 2, 9, 10, 9, 10}, {2, 2, 1, 11, 2, 5}, {2, 2, 15, 8, 1, 7}}}, }, - IndicesValues{5, 6, 2, 8}}, - ScatterNDUpdateLayerParams{ScatterNDUpdateShapes{{{-1, -1, -1, -1}, {{10, 9, 9, 11}, {7, 5, 3, 12}, {3, 4, 9, 8}}}, - {{2, 3}, {{2, 3}, {2, 3}, {2, 3}}}, - {{-1, -1}, {{2, 11}, {2, 12}, {2, 8}}}}, - IndicesValues{0, 1, 1, 2, 2, 2}}, + IndicesValues{5, 6, 2, 8} + }, ScatterNDUpdateLayerParams{ - ScatterNDUpdateShapes{{{{3, 10}, -1, {3, 9}, -1}, {{10, 9, 9, 11}, {7, 5, 3, 12}, {3, 4, 9, 8}}}, - {{2, 3}, {{2, 3}, {2, 3}, {2, 3}}}, - {{{2, 4}, -1}, {{2, 11}, {2, 12}, {2, 8}}}}, + ScatterNDUpdateShapes{ + {{-1, -1, -1, -1}, {{10, 9, 9, 11}, {7, 5, 3, 12}, {3, 4, 9, 8}}}, + {{2, 3}, {{2, 3}, {2, 3}, {2, 3}}}, + {{-1, -1}, {{2, 11}, {2, 12}, {2, 8}}} + }, + IndicesValues{0, 1, 1, 2, 2, 2} + }, + ScatterNDUpdateLayerParams{ + ScatterNDUpdateShapes{ + {{{3, 10}, -1, {3, 9}, -1}, {{10, 9, 9, 11}, {7, 5, 3, 12}, {3, 4, 9, 8}}}, + {{2, 3}, {{2, 3}, {2, 3}, {2, 3}}}, + {{{2, 4}, -1}, {{2, 11}, {2, 12}, {2, 8}}} + }, IndicesValues{0, 1, 1, 2, 2, 2}}, ScatterNDUpdateLayerParams{ - ScatterNDUpdateShapes{{{{3, 10}, {4, 11}, {3, 9}, {8, 15}}, {{10, 9, 9, 11}, {7, 5, 3, 12}, {3, 4, 9, 8}}}, - {{2, 3}, {{2, 3}, {2, 3}, {2, 3}}}, - {{{2, 4}, -1}, {{2, 11}, {2, 12}, {2, 8}}}}, + ScatterNDUpdateShapes{ + {{{3, 10}, {4, 11}, {3, 9}, {8, 15}}, {{10, 9, 9, 11}, {7, 5, 3, 12}, {3, 4, 9, 8}}}, + {{2, 3}, {{2, 3}, {2, 3}, {2, 3}}}, + {{{2, 4}, -1}, {{2, 11}, {2, 12}, {2, 8}}} + }, IndicesValues{0, 1, 1, 2, 2, 2}}, ScatterNDUpdateLayerParams{ - ScatterNDUpdateShapes{{{{3, 10}, {4, 11}, {3, 9}, {8, 15}}, {{10, 9, 9, 11}, {7, 5, 3, 12}, {3, 4, 9, 8}}}, - {{2, 3}, {{2, 3}, {2, 3}, {2, 3}}}, - {{{2, 4}, -1}, {{2, 11}, {2, 12}, {2, 8}}}}, + ScatterNDUpdateShapes{ + {{{3, 10}, {4, 11}, {3, 9}, {8, 15}}, {{10, 9, 9, 11}, {7, 5, 3, 12}, {3, 4, 9, 8}}}, + {{2, 3}, {{2, 3}, {2, 3}, {2, 3}}}, + {{{2, 4}, -1}, {{2, 11}, {2, 12}, {2, 8}}}}, IndicesValues{-1, -1, -1, -2, -2, -2}}, + // out of bounds indices + ScatterNDUpdateLayerParams{ + ScatterNDUpdateShapes{ + {{}, {{4, 8, 64, 1}}}, + {{}, {{1}}}, + {{}, {{8, 64, 1}}} + }, + IndicesValues{4}, // index is out of bounds + true + }, }; const std::vector inputPrecisions = { diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/single_layer_tests/reduce_ops.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/single_layer_tests/reduce_ops.cpp index 297d973a796dd0..9b7ae687e9c81d 100644 --- a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/single_layer_tests/reduce_ops.cpp +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/single_layer_tests/reduce_ops.cpp @@ -29,6 +29,12 @@ const std::vector> input_shapes = { std::vector{3, 5, 7, 9}, }; +const std::vector> input_shapes_0_dim = { + std::vector{2, 0, 4, 1}, + std::vector{8, 0, 4, 0}, + std::vector{0, 0, 0, 0}, +}; + const std::vector> input_shapes_one_axis = { std::vector{10, 20, 30, 40}, std::vector{3, 5, 7, 9}, @@ -167,6 +173,16 @@ const auto params_reduction_types = testing::Combine( testing::Values(ov::test::utils::DEVICE_CPU) ); +const auto params_empty_input = testing::Combine( + testing::ValuesIn(axes), + testing::Values(op_types[1]), + testing::ValuesIn(keep_dims), + testing::ValuesIn(reduction_types), + testing::Values(model_types[0]), + testing::ValuesIn(input_shapes_0_dim), + testing::Values(ov::test::utils::DEVICE_CPU) +); + const auto params_reduction_types_logical = testing::Combine( testing::Values(std::vector{0, 1, 3}), testing::Values(op_types[1]), @@ -250,6 +266,13 @@ INSTANTIATE_TEST_SUITE_P( ReduceOpsLayerTest::getTestCaseName ); +INSTANTIATE_TEST_SUITE_P( + smoke_Reduce_ReductionTypes_EmptyTensor, + ReduceOpsLayerTest, + params_empty_input, + ReduceOpsLayerTest::getTestCaseName +); + INSTANTIATE_TEST_SUITE_P( smoke_ReduceLogical_ReductionTypes, ReduceOpsLayerTest, diff --git a/src/plugins/intel_cpu/tools/commit_slider/tests/commit_slider_test.py b/src/plugins/intel_cpu/tools/commit_slider/tests/commit_slider_test.py index 2d342da339e810..c163a314b6d9e7 100644 --- a/src/plugins/intel_cpu/tools/commit_slider/tests/commit_slider_test.py +++ b/src/plugins/intel_cpu/tools/commit_slider/tests/commit_slider_test.py @@ -14,7 +14,8 @@ BenchmarkAppDataUnstable, BenchmarkAppDataStable, BenchmarkAppNoDegradationData,\ BenchmarkAppUnstableDevData, BenchmarkAppWrongPathData, BenchmarkAppPathFoundData,\ BenchmarkFirstFixedAppData, AcModeData, BenchmarkMetricData, CustomizedLogData, \ - MultiConfigData, ConfigMultiplicatorData, ConfigMultiplicatorWithKeyData + MultiConfigData, ConfigMultiplicatorData, ConfigMultiplicatorWithKeyData, \ + AcModeDataBitwise class CommitSliderTest(TestCase): @skip_commit_slider_devtest @@ -81,9 +82,9 @@ def testBmStable(self): self.assertEqual(breakCommit, actualCommit) @skip_commit_slider_devtest - def testACMode(self): + def testACModeBitwise(self): breakCommit, updatedData = getExpectedCommit( - AcModeData()) + AcModeDataBitwise()) actualCommit, _ = getActualCommit(updatedData) self.assertEqual(breakCommit, actualCommit) diff --git a/src/plugins/intel_cpu/tools/commit_slider/tests/test_data.py b/src/plugins/intel_cpu/tools/commit_slider/tests/test_data.py index 50249b1f24ab0f..53bd9d2c8a3dde 100644 --- a/src/plugins/intel_cpu/tools/commit_slider/tests/test_data.py +++ b/src/plugins/intel_cpu/tools/commit_slider/tests/test_data.py @@ -71,7 +71,8 @@ class TestCase(Enum): CustomizedLog = 15, MultiConfig = 16, ConfigMultiplicator = 17, - MultiConfigWithKey = 18 + MultiConfigWithKey = 18, + AcModeDataBitwise = 19 def requireTestData(self, reqLambda): # mapping json to test data holder @@ -190,6 +191,19 @@ def __init__(self): requireBinarySearchData ) +class AcModeDataBitwise(TestData): + def getTestCase(): + return TestData.TestCase.AcModeDataBitwise + + def getTestName(self): + return "ACMode" + + def __init__(self): + from test_util import requireBinarySearchData + self.requireTestData( + requireBinarySearchData + ) + class BenchmarkFirstFixedAppData(TestData): def getTestCase(): return TestData.TestCase.BmFirstFixed diff --git a/src/plugins/intel_cpu/tools/commit_slider/tests/tests_res/fbv_ac_mode_bitwise.cpp b/src/plugins/intel_cpu/tools/commit_slider/tests/tests_res/fbv_ac_mode_bitwise.cpp new file mode 100644 index 00000000000000..856438809f6acb --- /dev/null +++ b/src/plugins/intel_cpu/tools/commit_slider/tests/tests_res/fbv_ac_mode_bitwise.cpp @@ -0,0 +1,42 @@ +#include + +int main () { + const char *patchGenerator = R"V0G0N( +[ + { + "str": "std::cout << \"prefix\\Perplexity: 100.0% [FAILED: abs error = 9.118 | relative error = 0.3144]\\n\";", + "comment": "success_1" + }, + { + "str": "std::cout << \"prefix\\Perplexity: 100.0% [FAILED: abs error = 9.118 | relative error = 0.3144]\\n\";", + "comment": "success_2" + }, + { + "str": "std::cout << \"prefix\\Perplexity: 50.0% [FAILED: abs error = 9.118 | relative error = 0.3144]\\n\";", + "comment": "error_1", + "state": "BREAK" + }, + { + "str": "std::cout << \"prefix\\Perplexity: 50.0% [FAILED: abs error = 9.118 | relative error = 0.3144]\\n\";", + "comment": "error_2" + }, + { + "str": "std::cout << \"prefix\\Perplexity: 50.0% [FAILED: abs error = 9.118 | relative error = 0.3144]\\n\";", + "comment": "error_1" + }, + { + "str": "std::cout << \"prefix\\Perplexity: 50.0% [FAILED: abs error = 9.118 | relative error = 0.3144]\\n\";", + "comment": "error_2" + }, + { + "str": "std::cout << \"prefix\\Perplexity: 50.0% [FAILED: abs error = 9.118 | relative error = 0.3144]\\n\";", + "comment": "error_1" + }, + { + "str": "std::cout << \"prefix\\Perplexity: 50.0% [FAILED: abs error = 9.118 | relative error = 0.3144]\\n\";", + "comment": "error_2" + } +] +)V0G0N"; + return 0; +} \ No newline at end of file diff --git a/src/plugins/intel_cpu/tools/commit_slider/tests/tests_res/tests_res.json b/src/plugins/intel_cpu/tools/commit_slider/tests/tests_res/tests_res.json index 7f357753de17e5..8bdd8bec6a29c9 100644 --- a/src/plugins/intel_cpu/tools/commit_slider/tests/tests_res/tests_res.json +++ b/src/plugins/intel_cpu/tools/commit_slider/tests/tests_res/tests_res.json @@ -345,8 +345,27 @@ } } }, + "ACModeBitwise": { + "repoName": "ACModeBitwise", + "patchedFile": "tests_res/fbv_ac_mode_bitwise.cpp", + "testCfg": { + "appCmd" : "{appCmd}", + "appPath": "{appPath}", + "gitPath" : "{gitPath}", + "buildPath" : "{buildPath}", + "verboseOutput": false, + "runConfig" : { + "commitList" : { + "getCommitListCmd" : "git log {start}..{end} --boundary --pretty=\"%h\"" + }, + "mode" : "ac", + "traversal" : "firstFailedVersion", + "threshold": "80%" + } + } + }, "ACMode": { - "repoName": "BmBinarySearchUnstable", + "repoName": "ACMode", "patchedFile": "tests_res/fbv_ac_mode.cpp", "testCfg": { "appCmd" : "{appCmd}", @@ -359,7 +378,7 @@ "getCommitListCmd" : "git log {start}..{end} --boundary --pretty=\"%h\"" }, "mode" : "ac", - "traversal" : "firstFailedVersion", + "traversal" : "allBreaks", "threshold": "80%" } } diff --git a/src/plugins/intel_cpu/tools/commit_slider/utils/modes.py b/src/plugins/intel_cpu/tools/commit_slider/utils/modes.py index 4800d0854ae093..3340f1f597efd8 100644 --- a/src/plugins/intel_cpu/tools/commit_slider/utils/modes.py +++ b/src/plugins/intel_cpu/tools/commit_slider/utils/modes.py @@ -288,7 +288,7 @@ class AccuracyCheckerMode(Mode): def __init__(self, cfg): super().__init__(cfg) self.thresholdPattern = ":\s([0-9]*[.][0-9]*)%.*abs error" - self.breakThroughput = 0 + self.curMetric = None self.createCash() def prepareRun(self, list, cfg): @@ -317,29 +317,21 @@ def prepareRun(self, list, cfg): self.sampleThroughput = float(foundThroughput) return list - def checkCfg(self, cfg): - super().checkCfg(cfg) - if not ("threshold" in cfg["runConfig"]): - raise CfgError("Threshold is not configured") - else: - self.threshold = cfg["runConfig"]["threshold"] - self.threshold = float(self.threshold.strip('%')) - - def compareCommits(self, lCommit: str, rCommit: str, cfg: map): - leftThroughput = self.getPseudoMetric(lCommit, cfg) - rightThroughput = self.getPseudoMetric(rCommit, cfg) - isLeftGood = leftThroughput >= float(self.threshold) - isRightGood = rightThroughput >= float(self.threshold) - if not isRightGood: - self.breakThroughput = rightThroughput + leftMetric = self.getPseudoMetric(lCommit, cfg) + rightMetric = self.getPseudoMetric(rCommit, cfg) + isDiff = leftMetric != rightMetric + if isDiff: + self.curMetric = rightMetric curCommit = rCommit.replace('"', "") commitLogger = getCommitLogger(cfg, curCommit) - commitLogger.info("Current accuracy is {}%".format(rightThroughput)) + commitLogger.info("Current accuracy is {}%".format(rightMetric)) commitLogger.info( - "Commit is {status}".format(status=("bad" if isRightGood else "good")) + "Commit {status} from {c}".format( + status=("differs" if isDiff else "doesn't differ"), + c=lCommit) ) - return isLeftGood != isRightGood + return isDiff def getPseudoMetric(self, commit, cfg): commit = commit.replace('"', "") @@ -372,12 +364,12 @@ def getPseudoMetric(self, commit, cfg): return curThroughput def setOutputInfo(self, pathCommit): - pathCommit.breakThroughput = self.breakThroughput + pathCommit.metric = self.curMetric def getCommitInfo(self, commit): - return "{ci}, throughput = {d}".format( + return "{ci}, metric = {d}".format( ci=super().getCommitInfo(commit), - d=commit.breakThroughput) + d=commit.metric) class CompareBlobsMode(Mode): diff --git a/src/plugins/intel_gpu/include/intel_gpu/plugin/primitives_list.hpp b/src/plugins/intel_gpu/include/intel_gpu/plugin/primitives_list.hpp index ced915d25610e8..e234bc68de0750 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/plugin/primitives_list.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/plugin/primitives_list.hpp @@ -272,6 +272,7 @@ REGISTER_FACTORY(v13, BitwiseXor); REGISTER_FACTORY(v15, ROIAlignRotated); REGISTER_FACTORY(v15, BitwiseRightShift); REGISTER_FACTORY(v15, BitwiseLeftShift); +REGISTER_FACTORY(v15, SearchSorted); // --------------------------- Supported internal ops --------------------------- // REGISTER_FACTORY(internal, NonMaxSuppressionIEInternal); diff --git a/src/plugins/intel_gpu/include/intel_gpu/primitives/lstm.hpp b/src/plugins/intel_gpu/include/intel_gpu/primitives/lstm.hpp deleted file mode 100644 index 76f59e3448e694..00000000000000 --- a/src/plugins/intel_gpu/include/intel_gpu/primitives/lstm.hpp +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright (C) 2018-2024 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once -#include "primitive.hpp" -#include "activation.hpp" -#include -#include -#include "intel_gpu/graph/serialization/activation_serializer.hpp" - -namespace cldnn { - -/// @brief Weights orders -/// @details Specifies the order in which the weights are concatenated. -/// e.g. [i, o, f, z] : [input, output, forget, block] -/// ONNX order: iofz -/// Caffe order: ifoz -/// pyTorch order: izof -/// OV order: fizo -enum class lstm_weights_order { - iofz, - ifoz, - izof, - fizo -}; - -struct lstm_elt : public primitive_base { - CLDNN_DECLARE_PRIMITIVE(lstm_elt) - - lstm_elt() : primitive_base("", {}), clip(0), input_forget(0), offset_order(lstm_weights_order::iofz), direction(0) {} - - using vec_activation = std::vector; - using vec_activation_param = std::vector; - - /// @brief Constructs lstm layer. - /// @param id This primitive id. - /// @param input input primitive id. - /// @param input cell Primitive id containing cell data. Provide empty string if using lstm without cell values. - /// @param clip Clip threshold. Provide 0 if using lstm without activations clip threshold. - /// @param input_forget Provide 0 if using lstm without coupled input-forget gates. - /// @param offset_order. Order of the concatenated weights, recurrent, and bias. ONNX default is iofz [input, output, forget, block]. - /// @param direction default = 0, bidirectional = 1. - lstm_elt(const primitive_id& id, - const input_info& input, - const primitive_id& cell = "", - const float clip = 0, - const bool input_forget = 0, - const std::vector activations = {activation_func::logistic, - activation_func::hyperbolic_tan, - activation_func::hyperbolic_tan}, - const std::vector activation_params = {}, - const lstm_weights_order offset_order = lstm_weights_order::iofz, - const uint32_t direction = 0) - : primitive_base(id, {input}), - cell(cell), - clip(clip), - input_forget(input_forget), - activations(activations), - activation_params(activation_params), - offset_order(offset_order), - direction(direction) {} - - /// @brief Primitive id containing the initial value of the cell state data. - primitive_id cell; - /// @brief Cell clip threshold T. It is applied to the input of activations [-T, T]. No clip is applied if it is not specified. - float clip; - /// @brief Couple the input and forget gates if input_forget is 1. Default is 0. - bool input_forget; - /// @brief A list of 3 activation functions for the input, output, forget, cell, and hidden. - std::vector activations; - /// @brief Optional scaling values used by some activation functions. The values are consumed in the order of activation functions. - std::vector activation_params; - /// @brief Weights, recurrent weights, and biases order. [iofz] : ONNX, [ifoz] : Caffe - lstm_weights_order offset_order; - /// @brief direction default = 0, bidirectional = 1. - uint32_t direction; - - size_t hash() const override { - size_t seed = primitive::hash(); - seed = hash_combine(seed, clip); - seed = hash_combine(seed, input_forget); - seed = hash_range(seed, activations.begin(), activations.end()); - for (auto& act_param : activation_params) { - seed = hash_combine(seed, act_param.a); - seed = hash_combine(seed, act_param.b); - } - seed = hash_combine(seed, offset_order); - seed = hash_combine(seed, direction); - seed = hash_combine(seed, cell.empty()); - return seed; - } - - bool operator==(const primitive& rhs) const override { - if (!compare_common_params(rhs)) - return false; - - auto rhs_casted = downcast(rhs); - - bool act_params_eq = activation_params.size() == rhs_casted.activation_params.size(); - for (size_t i = 0; i < activation_params.size(); ++i) { - act_params_eq &= activation_params[i].a == rhs_casted.activation_params[i].a && - activation_params[i].b == rhs_casted.activation_params[i].b; - } - - #define cmp_fields(name) name == rhs_casted.name - return act_params_eq && - cmp_fields(clip) && - cmp_fields(input_forget) && - cmp_fields(activations) && - cmp_fields(offset_order) && - cmp_fields(direction) && - cmp_fields(cell.empty()); - #undef cmp_fields - } - - void save(BinaryOutputBuffer& ob) const override { - primitive_base::save(ob); - ob << cell; - ob << clip; - ob << input_forget; - ob << activations; - ob << activation_params; - ob << make_data(&offset_order, sizeof(lstm_weights_order)); - ob << direction; - } - - void load(BinaryInputBuffer& ib) override { - primitive_base::load(ib); - ib >> cell; - ib >> clip; - ib >> input_forget; - ib >> activations; - ib >> activation_params; - ib >> make_data(&offset_order, sizeof(lstm_weights_order)); - ib >> direction; - } - -protected: - std::vector get_dependencies() const override { - std::vector ret; - if (!cell.empty()) - ret.push_back(cell); - return ret; - } -}; - -} // namespace cldnn diff --git a/src/plugins/intel_gpu/include/intel_gpu/primitives/lstm_cell.hpp b/src/plugins/intel_gpu/include/intel_gpu/primitives/lstm_cell.hpp new file mode 100644 index 00000000000000..c53840d4bfd0c4 --- /dev/null +++ b/src/plugins/intel_gpu/include/intel_gpu/primitives/lstm_cell.hpp @@ -0,0 +1,24 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once +#include "primitive.hpp" +#include "activation.hpp" +#include +#include +#include "intel_gpu/graph/serialization/activation_serializer.hpp" +#include "rnn.hpp" + + +namespace cldnn { + +struct lstm_cell : public RNNParams { + CLDNN_DECLARE_PRIMITIVE(lstm_cell) + using vec_activation = std::vector; + using vec_activation_param = std::vector; + using RNNParams::RNNParams; + lstm_cell(const lstm_cell&) = default; + lstm_cell() : RNNParams() {} +}; +} // namespace cldnn diff --git a/src/plugins/intel_gpu/include/intel_gpu/primitives/rnn.hpp b/src/plugins/intel_gpu/include/intel_gpu/primitives/rnn.hpp new file mode 100644 index 00000000000000..a7c87c25bba235 --- /dev/null +++ b/src/plugins/intel_gpu/include/intel_gpu/primitives/rnn.hpp @@ -0,0 +1,188 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once +#include "primitive.hpp" +#include "activation.hpp" +#include +#include +#include +#include "intel_gpu/graph/serialization/activation_serializer.hpp" + +namespace cldnn { + +/// @brief Weights orders +/// @details Specifies the order in which the weights are concatenated. +/// e.g. [i, o, f, z] : [input, output, forget, block] +/// ONNX order: iofz +/// Caffe order: ifoz +/// pyTorch order: izof +/// OV order: fizo +enum class lstm_weights_order { + iofz, + ifoz, + izof, + fizo +}; + +template +struct RNNParams : public primitive_base { + RNNParams() : primitive_base("", {}) {} + RNNParams(const RNNParams&) = default; + RNNParams(const primitive_id& id, + const input_info& x, + const input_info& initial_hidden_state, + const input_info& initial_cell_state, + const input_info& W, + const input_info& R, + const input_info& B, + const input_info& seq_lenghts, + const float clip = 0, + bool input_forget = false, + const std::vector& activations = {activation_func::logistic, + activation_func::hyperbolic_tan, + activation_func::hyperbolic_tan}, + const std::vector& activation_params = {}, + const lstm_weights_order& offset_order = lstm_weights_order::iofz, + const ov::op::RecurrentSequenceDirection direction = ov::op::RecurrentSequenceDirection::FORWARD, + const int num_outputs = 1) + : primitive_base(id, {x}, num_outputs), + x(x), + initial_hidden_state(initial_hidden_state), + initial_cell_state(initial_cell_state), + W(W), + R(R), + B(B), + seq_lenghts(seq_lenghts), + clip(clip), + input_forget(input_forget), + activations(activations), + activation_params(activation_params), + offset_order(offset_order), + direction(direction) { + std::vector pids{initial_hidden_state.pid, initial_cell_state.pid, W.pid, R.pid, B.pid, seq_lenghts.pid}; + for (auto pid : pids) { + if (!pid.empty()) { + primitive_base::input.push_back(pid); + } + } + } + + input_info x; + input_info initial_hidden_state; + input_info initial_cell_state; + input_info W; + input_info R; + input_info B; + input_info seq_lenghts; + /// @brief Cell clip threshold T. It is applied to the input of activations [-T, T]. No clip is applied if it is not specified. + float clip; + bool input_forget; + /// @brief A list of 3 activation functions for the input, output, forget, cell, and hidden. + std::vector activations; + /// @brief Optional scaling values used by some activation functions. The values are consumed in the order of activation functions. + std::vector activation_params; + /// @brief Weights, recurrent weights, and biases order. [iofz] : ONNX, [ifoz] : Caffe + lstm_weights_order offset_order; + /// @brief direction of LSTMSequence - only FORWARD or REVERSE, currently BIDIRECTIONAL not supported + ov::op::RecurrentSequenceDirection direction; + + int num_directions() const { + return direction == ov::op::RecurrentSequenceDirection::BIDIRECTIONAL ? 2 : 1; + } + + size_t hash() const override { + size_t seed = primitive::hash(); + seed = hash_combine(seed, !x.pid.empty()); + seed = hash_combine(seed, !initial_hidden_state.pid.empty()); + seed = hash_combine(seed, !initial_cell_state.pid.empty()); + seed = hash_combine(seed, !seq_lenghts.pid.empty()); + seed = hash_combine(seed, !W.pid.empty()); + seed = hash_combine(seed, !R.pid.empty()); + seed = hash_combine(seed, !B.pid.empty()); + seed = hash_combine(seed, clip); + seed = hash_range(seed, activations.begin(), activations.end()); + for (auto& act_param : activation_params) { + seed = hash_combine(seed, act_param.a); + seed = hash_combine(seed, act_param.b); + } + seed = hash_combine(seed, offset_order); + seed = hash_combine(seed, direction); + return seed; + } + + bool operator==(const primitive& rhs) const override { + if (!primitive::compare_common_params(rhs)) + return false; + + auto rhs_casted = downcast(rhs); + bool act_params_eq = activation_params.size() == rhs_casted.activation_params.size(); + for (size_t i = 0; i < activation_params.size(); ++i) { + act_params_eq &= activation_params[i].a == rhs_casted.activation_params[i].a && + activation_params[i].b == rhs_casted.activation_params[i].b; + } + + #define cmp_fields(name) name == rhs_casted.name + return act_params_eq && + cmp_fields(x.pid.empty()) && + cmp_fields(initial_hidden_state.pid.empty()) && + cmp_fields(initial_cell_state.pid.empty()) && + cmp_fields(seq_lenghts.pid.empty()) && + cmp_fields(W.pid.empty()) && + cmp_fields(R.pid.empty()) && + cmp_fields(B.pid.empty()) && + cmp_fields(clip) && + cmp_fields(activations) && + cmp_fields(offset_order) && + cmp_fields(direction); + #undef cmp_fields + } + + void save(BinaryOutputBuffer& ob) const override { + primitive_base::save(ob); + ob << x; + ob << initial_hidden_state; + ob << initial_cell_state; + ob << W; + ob << R; + ob << B; + ob << seq_lenghts; + ob << clip; + ob << activations; + ob << activation_params; + ob << make_data(&offset_order, sizeof(lstm_weights_order)); + ob << make_data(&direction, sizeof(ov::op::RecurrentSequenceDirection)); + } + + void load(BinaryInputBuffer& ib) override{ + primitive_base::load(ib); + ib >> x; + ib >> initial_hidden_state; + ib >> initial_cell_state; + ib >> W; + ib >> R; + ib >> B; + ib >> seq_lenghts; + ib >> clip; + ib >> activations; + ib >> activation_params; + ib >> make_data(&offset_order, sizeof(lstm_weights_order)); + ib >> make_data(&direction, sizeof(ov::op::RecurrentSequenceDirection)); + } +}; + +struct lstm_seq : public RNNParams { + CLDNN_DECLARE_PRIMITIVE(lstm_seq) + using vec_activation = std::vector; + using vec_activation_param = std::vector; + using RNNParams::RNNParams; + lstm_seq() : RNNParams() { + weights = W.pid; + input = x.pid; + } + lstm_seq(const lstm_seq&) = default; + primitive_id input; + primitive_id weights; +}; +} //namespace cldnn diff --git a/src/plugins/intel_gpu/include/intel_gpu/primitives/rope.hpp b/src/plugins/intel_gpu/include/intel_gpu/primitives/rope.hpp index d7933e2180fe6f..a90caaa8a8cb9f 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/primitives/rope.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/primitives/rope.hpp @@ -43,6 +43,7 @@ struct rope : public primitive_base { seed = hash_combine(seed, config.input_trans0213); seed = hash_combine(seed, config.is_chatglm); seed = hash_combine(seed, config.support_2d_rope); + seed = hash_combine(seed, config.output_trans0213); seed = hash_combine(seed, config.is_interleaved); seed = hash_combine(seed, config.is_qwen); seed = hash_combine(seed, config.rotary_ndims); @@ -64,6 +65,7 @@ struct rope : public primitive_base { config.input_trans0213 == rhs_casted.config.input_trans0213 && config.is_chatglm == rhs_casted.config.is_chatglm && config.support_2d_rope == rhs_casted.config.support_2d_rope && + config.output_trans0213 == rhs_casted.config.output_trans0213 && config.is_interleaved == rhs_casted.config.is_interleaved && config.is_qwen == rhs_casted.config.is_qwen && config.rotary_ndims == rhs_casted.config.rotary_ndims && @@ -80,6 +82,7 @@ struct rope : public primitive_base { ob << config.input_trans0213; ob << config.is_chatglm; ob << config.support_2d_rope; + ob << config.output_trans0213; ob << config.is_interleaved; ob << config.is_qwen; ob << config.rotary_ndims; @@ -96,6 +99,7 @@ struct rope : public primitive_base { ib >> config.input_trans0213; ib >> config.is_chatglm; ib >> config.support_2d_rope; + ib >> config.output_trans0213; ib >> config.is_interleaved; ib >> config.is_qwen; ib >> config.rotary_ndims; diff --git a/src/plugins/intel_gpu/include/intel_gpu/primitives/search_sorted.hpp b/src/plugins/intel_gpu/include/intel_gpu/primitives/search_sorted.hpp new file mode 100644 index 00000000000000..4dfb5c87f8c58c --- /dev/null +++ b/src/plugins/intel_gpu/include/intel_gpu/primitives/search_sorted.hpp @@ -0,0 +1,54 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once +#include "primitive.hpp" + +namespace cldnn { + +/// @brief +/// @details +struct search_sorted : public primitive_base { + CLDNN_DECLARE_PRIMITIVE(search_sorted) + + search_sorted() : primitive_base("", {}) {} + + /// @brief Constructs search_sorted primitive. + /// @param id This primitive id. + /// @param sorted Sorted input. + /// @param values Values input. + /// @param right_mode Enable/Disable right mode(check specification for details).. + search_sorted(const primitive_id& id, const input_info& sorted, const input_info& values, bool right_mode) + : primitive_base(id, {sorted, values}), + right_mode(right_mode) {} + + /// @brief Enable/Disable right mode(check specification for details). + bool right_mode = false; + + size_t hash() const override { + size_t seed = primitive::hash(); + seed = hash_combine(seed, right_mode); + return seed; + } + + bool operator==(const primitive& rhs) const override { + if (!compare_common_params(rhs)) + return false; + + auto rhs_casted = downcast(rhs); + + return right_mode == rhs_casted.right_mode; + } + + void save(BinaryOutputBuffer& ob) const override { + primitive_base::save(ob); + ob << right_mode; + } + + void load(BinaryInputBuffer& ib) override { + primitive_base::load(ib); + ib >> right_mode; + } +}; +} // namespace cldnn diff --git a/src/plugins/intel_gpu/include/intel_gpu/runtime/internal_properties.hpp b/src/plugins/intel_gpu/include/intel_gpu/runtime/internal_properties.hpp index febcabd57efba0..199261772dcf2e 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/runtime/internal_properties.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/runtime/internal_properties.hpp @@ -56,6 +56,7 @@ static constexpr Property max_dynamic_batch{"DYN static constexpr Property nv12_two_inputs{"GPU_NV12_TWO_INPUTS"}; static constexpr Property buffers_preallocation_ratio{"GPU_BUFFERS_PREALLOCATION_RATIO"}; static constexpr Property max_kernels_per_batch{"GPU_MAX_KERNELS_PER_BATCH"}; +static constexpr Property use_onednn{"USE_ONEDNN"}; } // namespace intel_gpu } // namespace ov diff --git a/src/plugins/intel_gpu/src/graph/concatenation.cpp b/src/plugins/intel_gpu/src/graph/concatenation.cpp index b493bb217b1c32..3f1e1b03a2c564 100644 --- a/src/plugins/intel_gpu/src/graph/concatenation.cpp +++ b/src/plugins/intel_gpu/src/graph/concatenation.cpp @@ -114,7 +114,7 @@ concatenation_inst::typed_primitive_inst(network& network, concatenation_node co auto input_size = input_layout.get_dims(); auto output_size = output_layout.get_dims(); for (const auto& i : node.get_dependencies()) { - auto input_i_layout = i.first->get_output_layout(); + auto input_i_layout = i.first->get_output_layout(false, i.second); auto input_mem_size = input_i_layout.get_dims(); for (int64_t dim = 0; dim < static_cast(output_layout.get_rank()); ++dim) { if (dim == node.get_primitive()->axis) { diff --git a/src/plugins/intel_gpu/src/graph/debug_helper.cpp b/src/plugins/intel_gpu/src/graph/debug_helper.cpp index c2c41fdfab2373..b69d10e137010e 100644 --- a/src/plugins/intel_gpu/src/graph/debug_helper.cpp +++ b/src/plugins/intel_gpu/src/graph/debug_helper.cpp @@ -51,11 +51,13 @@ void dump(memory::ptr mem, stream& stream, std::ofstream& file_stream, bool dump if (tmp_size == size) { file_stream << "shape: " << size.to_string() << " "; file_stream << "(count: " << size.count() + << ", addr: " << mem->buffer_ptr() << ", original format: " << cldnn::fmt_to_str(mem->get_layout().format) << ")" << (dump_raw ? " raw data" : "") << std::endl; } else { file_stream << "shape: " << tmp_size.to_string() << " "; file_stream << "(count: " << tmp_size.count() + << ", addr: " << mem->buffer_ptr() << ", original format: " << cldnn::fmt_to_str(mem->get_layout().format) << ", original shape: " << size.to_string() << ")" << (dump_raw ? " raw data" : "") << std::endl; diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/post_optimize_weights.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/post_optimize_weights.cpp index 9805b45ad005ed..377e0eef2402f0 100644 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/post_optimize_weights.cpp +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/post_optimize_weights.cpp @@ -9,7 +9,10 @@ #include "convolution_inst.h" #include "deconvolution_inst.h" #include "fully_connected_inst.h" +#include "lstm_seq_inst.h" #include "intel_gpu/runtime/format.hpp" +#include "permute_inst.h" +#include "crop_inst.h" #ifdef ENABLE_ONEDNN_FOR_GPU #include "graph/impls/onednn/utils.hpp" #endif // ENABLE_ONEDNN_FOR_GPU @@ -18,10 +21,17 @@ namespace cldnn { post_optimize_weights::post_optimize_weights(reorder_factory& rf_ref) : base_pass("post_optimize_weights"), _rf(rf_ref) {} -template post_optimize_weights::weights_bias_offset post_optimize_weights::get_weights_bias_offset(const T& node) { +template +post_optimize_weights::weights_bias_offset post_optimize_weights::get_weights_bias_offset(const T& node) { return weights_bias_offset(node.get_primitive()->input.size(), program_helpers::wrap_if_single(node.get_primitive()->weights).size()); } +template <> +post_optimize_weights::weights_bias_offset post_optimize_weights::get_weights_bias_offset(const lstm_seq_node& node) { + const int W_idx = 3; + return weights_bias_offset(W_idx, 3); +} + // function which prepares given primitive for weights optimization template void post_optimize_weights::optimize_weights(T& node, program& p) { @@ -109,15 +119,26 @@ void post_optimize_weights::optimize_weights(T& node, program& p) { set_implementation(weights_reorder_node); } } else { - auto weights_reorder = _rf.get_weights_reorder(prev_node.id(), weights_reorder_params); - // insert new weights reorder node to topology - p.add_intermediate(weights_reorder.first, node, i, !weights_reorder.second); - // set weights reorder's node output layout and implementation - auto& weights_reorder_node = node.get_dependency(i); - weights_reorder_node.get_output_layout(false); + if (node.type() == lstm_seq::type_id()) { + program_node& prev_node = node.get_dependency(i); + if (i == 5) { + add_lstm_bias_reorder(prev_node.id(), weights_reorder_params, p, prev_node, node); + } else { + add_lstm_weights_reorder(prev_node.id(), weights_reorder_params, p, prev_node, node, i); + } + auto& weights_reorder_node = node.get_dependency(i); + weights_reorder_node.get_output_layout(false); + } else { + auto weights_reorder = _rf.get_weights_reorder(prev_node.id(), weights_reorder_params); + // insert new weights reorder node to topology + p.add_intermediate(weights_reorder.first, node, i, !weights_reorder.second); + // set weights reorder's node output layout and implementation + auto& weights_reorder_node = node.get_dependency(i); + weights_reorder_node.get_output_layout(false); - if (!weights_reorder.second) { - set_implementation(weights_reorder_node); + if (!weights_reorder.second) { + set_implementation(weights_reorder_node); + } } } } @@ -126,7 +147,112 @@ void post_optimize_weights::optimize_weights(T& node, program& p) { node.set_output_layout(output_layout, false); } +void post_optimize_weights::select_implementation(program& p, program_node& node) { + node.set_selected_impl(node.type()->create_impl(node)); + if (auto impl = node.get_selected_impl()) { + auto params = node.get_kernel_impl_params(); + p.get_kernels_cache().add_kernels_source(*params, impl->get_kernels_source()); + } +} + +void post_optimize_weights::add_lstm_weights_reorder(primitive_id input_id, std::shared_ptr reorder_params, program& p, \ + cldnn::program_node& prev, cldnn::program_node& node, size_t i) { + OPENVINO_ASSERT(reorder_params != nullptr, "[GPU] WeightsReorderParams is not initialized."); + std::string reorder_id = input_id + "_reo_" + std::to_string(i); + const auto dir_num = static_cast(reorder_params->get_input_layout().get_shape()[0]); + auto hiddenSize = reorder_params->get_input_layout().get_shape()[1] / 4; + auto inputSize = static_cast(reorder_params->get_input_layout().get_shape()[2]); + int size_third; + const int W_idx = 3; + if (i == W_idx) { + size_third = inputSize; + } else { + size_third = static_cast(hiddenSize); + } + auto cropSizeR = cldnn::tensor{dir_num, static_cast(hiddenSize), 1, size_third}; + std::string crop_id_b = input_id + "_c"; + auto get_crop_node = [&](int cropNum) -> cldnn::program_node& { + auto crop_id = primitive_id(crop_id_b + std::to_string(cropNum)); + auto crop_prim = std::make_shared(crop_id, reorder_id, cropSizeR, cldnn::tensor{0, static_cast(cropNum*hiddenSize), 0, 0}); + return p.get_or_create(crop_prim); + }; + + auto& crop0_node = get_crop_node(0); + auto& crop1_node = get_crop_node(1); + auto crop2_id = primitive_id(crop_id_b + std::to_string(2)); + auto crop2_prim = std::make_shared(crop2_id, reorder_id, cldnn::tensor{dir_num, static_cast(2*hiddenSize), 1, size_third}, + cldnn::tensor{0, static_cast(2*hiddenSize), 0, 0}); + auto& crop2_node = p.get_or_create(crop2_prim); + std::vector con_input{input_info(crop_id_b + "1"), input_info(crop_id_b + "0"), input_info(crop_id_b + "2")}; + cldnn::primitive_id concat_id{input_id + "cont"}; + auto con = std::make_shared(concat_id, con_input, 1); + auto& con_node = p.get_or_create(con); + p.add_intermediate(con_node, node, prev, true); + p.add_intermediate(crop1_node, con_node, prev, true); + p.add_connection(prev, crop0_node, 0); + p.add_connection(prev, crop2_node, 0); + p.add_connection(crop0_node, con_node, 0); + p.add_connection(crop2_node, con_node, 0); + std::string permute_id = input_id + "_perx"; + std::vector ord{0, 2, 1}; + auto permute = std::make_shared(permute_id, input_info{concat_id}, ord); + auto& permute_node = p.get_or_create(permute); + p.add_intermediate(permute_node, node, con_node, true); + auto set_implementation_and_output = [this, &p](program_node& node) { + node.get_output_layout(false); + select_implementation(p, node); + p.mark_if_constant(node); + node.recalc_output_layout(false); + }; + set_implementation_and_output(crop1_node); + set_implementation_and_output(crop0_node); + set_implementation_and_output(crop2_node); + set_implementation_and_output(con_node); + set_implementation_and_output(permute_node); +} + +void post_optimize_weights::add_lstm_bias_reorder(primitive_id input_id, std::shared_ptr reorder_params, program& p, \ + cldnn::program_node& prev, cldnn::program_node& node) { + OPENVINO_ASSERT(reorder_params != nullptr, "[GPU] WeightsReorderParams is not initialized."); + const auto dir_num = static_cast(reorder_params->get_input_layout().get_shape()[0]); + auto hiddenSize = reorder_params->get_output_layout().get_shape()[1] / 4; + auto cropSize = cldnn::tensor{dir_num, static_cast(hiddenSize), 1, 1}; + std::string crop_id_b = input_id + "_c"; + auto get_crop_node = [&](int cropNum) -> cldnn::program_node& { + auto crop_id = primitive_id(crop_id_b + std::to_string(cropNum)); + auto crop_prim = std::make_shared(crop_id, input_id, cropSize, cldnn::tensor{0, static_cast(cropNum*hiddenSize), 0, 0}); + return p.get_or_create(crop_prim); + }; + auto& crop0_node = get_crop_node(0); + auto& crop1_node = get_crop_node(1); + auto crop2_id = primitive_id(crop_id_b + std::to_string(2)); + auto crop2_prim = std::make_shared(crop2_id, input_id, cldnn::tensor{dir_num, static_cast(2*hiddenSize), 1, 1}, + cldnn::tensor{0, static_cast(2*hiddenSize), 0, 0}); + auto& crop2_node = p.get_or_create(crop2_prim); + std::vector con_input{input_info(crop1_node.id()), input_info(crop0_node.id()), input_info(crop2_node.id())}; + cldnn::primitive_id concat_id{input_id + "concat"}; + auto con = std::make_shared(concat_id, con_input, 1); + auto& con_node = p.get_or_create(con); + p.add_intermediate(con_node, node, prev, true); + p.add_intermediate(crop1_node, con_node, prev, true); + p.add_connection(prev, crop0_node, 0); + p.add_connection(prev, crop2_node, 0); + p.add_connection(crop0_node, con_node, 0); + p.add_connection(crop2_node, con_node, 0); + auto set_implementation_and_output = [this, &p](program_node& node) { + node.get_output_layout(false); + select_implementation(p, node); + p.mark_if_constant(node); + node.recalc_output_layout(false); + }; + set_implementation_and_output(crop0_node); + set_implementation_and_output(crop1_node); + set_implementation_and_output(crop2_node); + set_implementation_and_output(con_node); +} + void post_optimize_weights::run(program& p) { + bool found_lstm = false; for (auto& node : p.get_processing_order()) { if (node->is_type()) { optimize_weights(node->as(), p); @@ -134,8 +260,13 @@ void post_optimize_weights::run(program& p) { optimize_weights(node->as(), p); } else if (node->is_type()) { optimize_weights(node->as(), p); + } else if (node->is_type()) { + optimize_weights(node->as(), p); + found_lstm = true; } } + if (found_lstm) + p.get_processing_order().calc_processing_order(p); } } // namespace cldnn diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_buffer_fusing.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_buffer_fusing.cpp index e94714c84fdebf..de7f51b071ae53 100644 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_buffer_fusing.cpp +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_buffer_fusing.cpp @@ -17,11 +17,12 @@ #include "depth_to_space_inst.h" #include "resample_inst.h" #include "loop_inst.h" -#include "lstm_elt_inst.h" +#include "lstm_cell_inst.h" #include "strided_slice_inst.h" #include "shape_of_inst.h" #include "non_max_suppression_inst.h" #include "experimental_detectron_roi_feature_extractor_inst.hpp" +#include "lstm_seq_inst.h" #include "border_inst.h" #include "pass_manager.h" @@ -504,6 +505,8 @@ bool crop_in_place_optimization::match(const program_node& node, } if (user->is_type() && user->get_dependency_index(node) == 0) return false; + if (user->is_type() || user->is_type()) + return false; } // do not optimize crop, that must be calculated in propagate_constants @@ -519,10 +522,6 @@ bool crop_in_place_optimization::match(const program_node& node, return false; if (node.get_users().size() > 0) { - if (node.get_program().is_body_program() && node.get_dependency(0).is_type()) { - return false; - } - GPU_DEBUG_GET_INSTANCE(debug_config); GPU_DEBUG_IF(debug_config->disable_runtime_buffer_fusing && node.is_dynamic()) { return false; diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp index c323109850c489..60d1e8aa7e10b7 100644 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp @@ -439,7 +439,7 @@ void prepare_primitive_fusing::fuse_simple_primitives(program &p) { }; auto conv_supports_fusings = [&](convolution_node& node) -> bool { - if (lo.get_optimization_attributes().use_onednn_impls == 1 && + if (lo.has_all_enabled_onednn_impls_optimization_attribute() && lo.get_preferred_impl_type(node, format::byxf /*dummy value to disable format checking*/) == impl_types::onednn) { return true; } @@ -491,7 +491,7 @@ void prepare_primitive_fusing::fuse_simple_primitives(program &p) { }; auto fc_supports_fusings = [&](fully_connected_node& node) -> bool { - if (lo.get_optimization_attributes().use_onednn_impls && + if (lo.has_all_enabled_onednn_impls_optimization_attribute() && lo.get_preferred_impl_type(node, format::any /*dummy*/) == impl_types::onednn) { return true; } else { @@ -589,7 +589,7 @@ void prepare_primitive_fusing::fuse_simple_primitives(program &p) { // Do not fuse if the estimated format is fs_b_yx_fsv32 because the optimized kernel does not support fusion if (out_layout.data_type == data_types::f16 && out_layout.is_static() && out_layout.batch() > 1 && ((lo.get_optimization_attributes().fs_b_yx_fsv32_network && - !lo.get_optimization_attributes().use_onednn_impls && !has_reorder_behind_mvn()) || + !lo.has_all_enabled_onednn_impls_optimization_attribute() && !has_reorder_behind_mvn()) || out_layout.format == format::fs_b_yx_fsv32)) { return false; } @@ -665,7 +665,7 @@ void prepare_primitive_fusing::fuse_simple_primitives(program &p) { if (input.in_shape_of_subgraph || node->in_shape_of_subgraph) return; - if (lo.get_optimization_attributes().use_onednn_impls) { + if (lo.has_all_enabled_onednn_impls_optimization_attribute()) { if (input.is_type() || input.is_type()) return; auto additional_params_input = activation_node.get_primitive()->additional_params_input; @@ -768,7 +768,7 @@ void prepare_primitive_fusing::fuse_simple_primitives(program &p) { return; // Onednn reorder does not support eltwise nor binary post operation - if (lo.get_optimization_attributes().use_onednn_impls && input.is_type()) { + if (lo.has_all_enabled_onednn_impls_optimization_attribute() && input.is_type()) { return; } @@ -809,7 +809,7 @@ void prepare_primitive_fusing::fuse_simple_primitives(program &p) { (lo.should_select_b_fs_yx_fsv16_layout(input_data.as(), input_data.get_input_layout(1)) && !is_grouped_conv(input_data.as())) || // Avoid fusing to b_fs_yx_fsv16 (and similar) kernels - lo.get_optimization_attributes().use_onednn_impls || + (lo.has_all_enabled_onednn_impls_optimization_attribute()) || (in_dt_is_i8_u8 && out_dt_is_i8_u8)); should_fuse |= input_data.is_type() && quantize_node.get_scale_shift_opt(); @@ -1067,7 +1067,7 @@ void prepare_primitive_fusing::fuse_simple_primitives(program &p) { } } - if (lo.get_optimization_attributes().use_onednn_impls && lo.is_primitive_implemented_for_onednn(*fused_node)) { + if (lo.has_all_enabled_onednn_impls_optimization_attribute() && lo.is_primitive_implemented_for_onednn(*fused_node)) { auto eltw_in_size = peer_node->get_output_layout(); if (eltw_in_size.is_dynamic() // this whitelist condition is temporarily and to be relaxed soon. diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/remove_redundant_reorders.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/remove_redundant_reorders.cpp index 28ee84c4a4ec02..1e5f943600fc05 100644 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/remove_redundant_reorders.cpp +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/remove_redundant_reorders.cpp @@ -489,7 +489,7 @@ void remove_redundant_reorders::run(program& p) { (dep.get_output_layout().format == format::b_fs_yx_fsv16 || dep.get_output_layout().format == format::bfyx || (dep.get_output_layout().format == format::fs_b_yx_fsv32 && - !lo.get_optimization_attributes().use_onednn_impls)); + !lo.has_all_enabled_onednn_impls_optimization_attribute())); auto convert_color_opt = usr->is_type() && prim_desc->has_surface_input(); diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/reorder_inputs.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/reorder_inputs.cpp index 4f15800c70970b..b29be318593348 100644 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/reorder_inputs.cpp +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/reorder_inputs.cpp @@ -56,9 +56,9 @@ std::map get_preferred_formats(program& p, layout_o onednn_impls_counter++; } - if (lo.get_optimization_attributes().use_onednn_impls && onednn_impls_counter < 1) { + if (!lo.is_empty_onednn_impls_optimization_attribute() && onednn_impls_counter < 1) { should_update_fmt_map = true; - lo.set_optimization_attribute(layout_optimizer::optimization_attributes_type::use_onednn_impls, 0); + lo.clear_onednn_impls_optimization_attribute(); GPU_DEBUG_LOG << "Disable oneDNN implementations globally" << std::endl; } diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/select_preferred_formats.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/select_preferred_formats.cpp index 8a1197dfb843a6..fcd6dab33754fd 100644 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/select_preferred_formats.cpp +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/select_preferred_formats.cpp @@ -69,7 +69,7 @@ void select_preferred_formats::run(program& p) { #ifdef ENABLE_ONEDNN_FOR_GPU auto& engine = p.get_engine(); - if (p.get_layout_optimizer().get_optimization_attributes().use_onednn_impls) { + if (!p.get_layout_optimizer().is_empty_onednn_impls_optimization_attribute()) { engine.create_onednn_engine(p.get_config()); } #endif // ENABLE_ONEDNN_FOR_GPU diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.cpp index 409fd824063da6..0a999a5a124d3b 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.cpp @@ -309,6 +309,8 @@ kernel_selector::data_layout to_data_layout(format f) { return kernel_selector::data_layout::bfzyx; case format::bzyxf: return kernel_selector::data_layout::bzyxf; + case format::ybfx: + return kernel_selector::data_layout::ybfx; case format::fs_b_yx_fsv32: return kernel_selector::data_layout::fs_b_yx_fsv32; case format::bfwzyx: diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/lstm_cell.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl/lstm_cell.cpp new file mode 100644 index 00000000000000..a41cd1065122de --- /dev/null +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/lstm_cell.cpp @@ -0,0 +1,93 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "primitive_base.hpp" + +#include "lstm_cell_inst.h" +#include "lstm/lstm_cell_and_seq_kernel_selector.h" +#include "lstm/lstm_kernel_base.h" +#include "openvino/op/lstm_cell.hpp" +#include "lstm_cell.hpp" + +namespace cldnn { +namespace ocl { + +struct lstm_cell_impl : typed_primitive_impl_ocl { + using parent = typed_primitive_impl_ocl; + using parent::parent; + using kernel_selector_t = kernel_selector::lstm_cell_and_seq_kernel_selector; + using kernel_params_t = kernel_selector::lstm_params; + + DECLARE_OBJECT_TYPE_SERIALIZATION(cldnn::ocl::lstm_cell_impl) + + std::unique_ptr clone() const override { + return make_unique(*this); + } + +protected: + kernel_arguments_data get_arguments(const typed_primitive_inst& instance) const override { + kernel_arguments_data args; + for (size_t i = 0; i < instance.inputs_memory_count(); i++) { + args.inputs.push_back(instance.input_memory_ptr(i)); + } + + for (size_t i = 0; i < instance.outputs_memory_count(); i++) { + args.outputs.push_back(instance.output_memory_ptr(i)); + } + return args; + } + +public: + static kernel_params_t get_kernel_params(const kernel_impl_params& impl_param) { + const auto& primitive = impl_param.typed_desc(); + auto params = get_default_params(impl_param); + for (size_t i = 1; i < 6; ++i) { + params.inputs.push_back(convert_data_tensor(impl_param.get_input_layout(i))); + } + + if (!primitive->activations.empty()) { + auto a_sz = primitive->activations.size(); + auto param_sz = primitive->activation_params.size(); + OPENVINO_ASSERT(param_sz == 0 || a_sz == param_sz, "[GPU] Unexpected activation params count in lstm_cell impl: ", param_sz); + for (size_t i = 0; i < a_sz; i++) { + params.activations.emplace_back(get_kernel_selector_activation_param(primitive->activations[i]), + param_sz ? primitive->activation_params[i].a : 0.0f, + param_sz ? primitive->activation_params[i].b : 0.0f); + } + } + + if (primitive->clip > 0.0f) { + params.activations.emplace_back(get_kernel_selector_activation_param(activation_func::clamp), -primitive->clip, primitive->clip); + } + + params.SetOffsetOrder(static_cast(primitive->offset_order)); + params.clip = primitive->clip; + params.direction = primitive->direction; + + return params; + } + + static kernel_impl_params static_canonicalize_shapes(const kernel_impl_params& impl_params) { + if (impl_params.get_input_layout().get_partial_shape().size() != 3) { + return primitive_impl::static_canonicalize_shapes(impl_params); + } + auto updated_impl_params = canonicalize_fused_shapes(impl_params); + return updated_impl_params; + } + + kernel_impl_params canonicalize_shapes(const kernel_impl_params& impl_params) const override { + return static_canonicalize_shapes(impl_params); + } +}; + +std::unique_ptr LSTMCellImplementationManager::create_impl(const program_node& node, const kernel_impl_params& params) const { + OPENVINO_ASSERT(node.is_type()); + return typed_primitive_impl_ocl::create(static_cast(node), params); +} + +} // namespace ocl +} // namespace cldnn + +BIND_BINARY_BUFFER_WITH_TYPE(cldnn::ocl::lstm_cell_impl) +BIND_BINARY_BUFFER_WITH_TYPE(cldnn::lstm_cell) diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/lstm_cell.hpp b/src/plugins/intel_gpu/src/graph/impls/ocl/lstm_cell.hpp new file mode 100644 index 00000000000000..731bacf2e17e4f --- /dev/null +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/lstm_cell.hpp @@ -0,0 +1,47 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "lstm_cell_inst.h" +#include "impls/registry/implementation_manager.hpp" +#include "intel_gpu/runtime/layout.hpp" + +#include +namespace cldnn { +namespace ocl { + +struct LSTMCellImplementationManager: public ImplementationManager { + OV_GPU_PRIMITIVE_IMPL("ocl::lstm_cell") + LSTMCellImplementationManager(shape_types shape_type, ValidateFunc vf = nullptr) : ImplementationManager(impl_types::ocl, shape_type, vf) {} + + std::unique_ptr create_impl(const program_node& node, const kernel_impl_params& params) const override; + + bool validate_impl(const program_node& node) const override { + assert(node.is_type()); + + const auto& input_layout = node.get_input_layout(0); + const auto& output_layout = node.get_output_layout(0); + + auto input_fmt = input_layout.format; + auto output_fmt = output_layout.format; + auto in_dt = input_layout.data_type; + auto out_dt = output_layout.data_type; + static const std::vector supported_formats = { + format::bfyx, + format::fyxb, + }; + static const std::vector supported_data_types = { + data_types::f32, + data_types::f16, + }; + + if (!one_of(in_dt, supported_data_types) || !one_of(out_dt, supported_data_types)) { + return false; + } + + return one_of(input_fmt.value, supported_formats) && one_of(output_fmt.value, supported_formats); + } +}; + +} // namespace ocl +} // namespace cldnn diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/lstm_elt.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl/lstm_elt.cpp deleted file mode 100644 index 5de12d83fdbab3..00000000000000 --- a/src/plugins/intel_gpu/src/graph/impls/ocl/lstm_elt.cpp +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright (C) 2018-2024 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "primitive_base.hpp" - -#include "lstm_elt_inst.h" -#include "lstm/lstm_elt_kernel_selector.h" -#include "lstm/lstm_elt_kernel_base.h" - -namespace cldnn { -namespace ocl { - -struct lstm_elt_impl : typed_primitive_impl_ocl { - using parent = typed_primitive_impl_ocl; - using parent::parent; - using kernel_selector_t = kernel_selector::lstm_elt_kernel_selector; - using kernel_params_t = kernel_selector::lstm_elt_params; - - DECLARE_OBJECT_TYPE_SERIALIZATION(cldnn::ocl::lstm_elt_impl) - - std::unique_ptr clone() const override { - return make_deep_copy(*this); - } - -protected: - kernel_arguments_data get_arguments(const typed_primitive_inst& instance) const override { - kernel_arguments_data args = parent::get_arguments(instance); - - args.cell = instance.cell_term() ? instance.cell_memory() : nullptr; - args.outputs = { instance.output_memory_ptr() }; - - return args; - } - -public: - static kernel_params_t get_kernel_params(const kernel_impl_params& impl_param) { - const auto& primitive = impl_param.typed_desc(); - auto params = get_default_params(impl_param); - - if (!primitive->cell.empty()) { - const auto& cell_idx = 1; - const auto& cell_layout = impl_param.input_layouts[cell_idx]; - params.SetCell(convert_data_tensor(cell_layout)); - // TODO: make a generic function to get the direction - if (cell_layout.spatial(1) > 1) { - params.cell_direction = primitive->direction; - } - } - - if (!primitive->activations.empty()) { - auto a_sz = primitive->activations.size(); - auto param_sz = primitive->activation_params.size(); - OPENVINO_ASSERT(param_sz == 0|| a_sz == param_sz, "[GPU] Unexpected activation params count in lstm_elt impl: ", param_sz); - for (size_t i = 0; i < a_sz; i++) { - params.activations.emplace_back(get_kernel_selector_activation_param(primitive->activations[i]), - param_sz ? primitive->activation_params[i].a : 0.0f, - param_sz ? primitive->activation_params[i].b : 0.0f); - } - } - - if (primitive->clip > 0.0f) { - params.activations.emplace_back(get_kernel_selector_activation_param(activation_func::clamp), -primitive->clip, primitive->clip); - } - - params.SetOffsetOrder(static_cast(primitive->offset_order)); - params.clip = primitive->clip; - params.input_forget = primitive->input_forget; - params.direction = primitive->direction; - - return params; - } - - static kernel_impl_params static_canonicalize_shapes(const kernel_impl_params& impl_params) { - if (impl_params.get_input_layout().get_partial_shape().size() != 2) { - return primitive_impl::static_canonicalize_shapes(impl_params); - } - auto updated_impl_params = canonicalize_fused_shapes(impl_params); - - auto& input_layout = updated_impl_params.input_layouts[0]; - auto& weights_layout = updated_impl_params.input_layouts[1]; - auto& output_layout = updated_impl_params.output_layouts[0]; - - auto input_pshape = input_layout.get_partial_shape(); - auto weights_pshape = weights_layout.get_partial_shape(); - auto output_pshape = output_layout.get_partial_shape(); - - auto lstm_input_size = static_cast(input_pshape[1].get_length()); - auto lstm_batch_size = static_cast(input_pshape[0].get_length()); - auto lstm_hidden_size = static_cast(lstm_input_size / 4); - - GPU_DEBUG_LOG << "lstm_input_size : " << lstm_input_size << std::endl; - GPU_DEBUG_LOG << "lstm_batch_size : " << lstm_batch_size << std::endl; - GPU_DEBUG_LOG << "lstm_hidden_size : " << lstm_hidden_size << std::endl; - - GPU_DEBUG_LOG << "origin input_pshape : " << input_layout.to_short_string() << std::endl; - GPU_DEBUG_LOG << "origin weights_layout : " << weights_layout.to_short_string() << std::endl; - - input_pshape = {lstm_batch_size, 1, 1, lstm_input_size}; - input_layout.set_partial_shape(input_pshape); - - weights_pshape = {lstm_batch_size, 1, 1, lstm_hidden_size}; // {batch, direction, 1, hidden_size} - weights_layout.format = format::adjust_to_rank(weights_layout.format, weights_pshape.size()); - weights_layout.set_partial_shape(weights_pshape); - - updated_impl_params.weights_layout = weights_layout; - - GPU_DEBUG_LOG << "input_layout : " << input_layout.to_short_string() << std::endl; - GPU_DEBUG_LOG << "weights_layout : " << weights_layout.to_short_string() << std::endl; - GPU_DEBUG_LOG << "output_layout : " << output_layout.to_short_string() << std::endl; - - OPENVINO_ASSERT(input_pshape.size() == 4 && weights_pshape.size() == 4, "input and weights shape should be rank 4"); - return updated_impl_params; - } - - kernel_impl_params canonicalize_shapes(const kernel_impl_params& impl_params) const override { - return static_canonicalize_shapes(impl_params); - } -}; - -namespace detail { - -attach_lstm_elt_impl::attach_lstm_elt_impl() { - implementation_map::add(impl_types::ocl, typed_primitive_impl_ocl::create, { - std::make_tuple(data_types::f32, format::bfyx), - std::make_tuple(data_types::f16, format::bfyx), - std::make_tuple(data_types::f32, format::fyxb), - std::make_tuple(data_types::f16, format::fyxb), - }); -} - -} // namespace detail -} // namespace ocl -} // namespace cldnn - -BIND_BINARY_BUFFER_WITH_TYPE(cldnn::ocl::lstm_elt_impl) -BIND_BINARY_BUFFER_WITH_TYPE(cldnn::lstm_elt) diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/register.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl/register.cpp index 2597e419e66a41..2a38d20ac8c9bc 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl/register.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/register.cpp @@ -39,7 +39,6 @@ void register_implementations() { REGISTER_OCL(kv_cache); REGISTER_OCL(paged_attention); REGISTER_OCL(lrn); - REGISTER_OCL(lstm_elt); REGISTER_OCL(multiclass_nms); REGISTER_OCL(multinomial); REGISTER_OCL(mutable_data); @@ -88,6 +87,7 @@ void register_implementations() { REGISTER_OCL(unique_gather); REGISTER_OCL(scaled_dot_product_attention); REGISTER_OCL(rope); + REGISTER_OCL(search_sorted); } } // namespace ocl diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/register.hpp b/src/plugins/intel_gpu/src/graph/impls/ocl/register.hpp index d4b08b5154ef4b..c65a23822a6922 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl/register.hpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/register.hpp @@ -113,7 +113,6 @@ REGISTER_OCL(group_normalization); REGISTER_OCL(kv_cache); REGISTER_OCL(paged_attention); REGISTER_OCL(lrn); -REGISTER_OCL(lstm_elt); REGISTER_OCL(multiclass_nms); REGISTER_OCL(multinomial); REGISTER_OCL(mutable_data); @@ -162,6 +161,7 @@ REGISTER_OCL(unique_count); REGISTER_OCL(unique_gather); REGISTER_OCL(scaled_dot_product_attention); REGISTER_OCL(rope); +REGISTER_OCL(search_sorted); #undef REGISTER_OCL diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/rnn_seq.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl/rnn_seq.cpp new file mode 100644 index 00000000000000..3fb8ae13d3baa4 --- /dev/null +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/rnn_seq.cpp @@ -0,0 +1,94 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "primitive_base.hpp" + +#include "lstm_seq_inst.h" +#include "rnn_seq.hpp" +#include "lstm/lstm_cell_and_seq_kernel_selector.h" +#include "lstm/lstm_kernel_base.h" +#include "openvino/op/lstm_sequence.hpp" +#include "impls/registry/implementation_manager.hpp" + +namespace cldnn { +namespace ocl { + +struct rnn_seq_impl : typed_primitive_impl_ocl { + using parent = typed_primitive_impl_ocl; + using parent::parent; + using kernel_selector_t = kernel_selector::lstm_cell_and_seq_kernel_selector; + using kernel_params_t = kernel_selector::lstm_params; + + DECLARE_OBJECT_TYPE_SERIALIZATION(cldnn::ocl::rnn_seq_impl) + + std::unique_ptr clone() const override { + return make_unique(*this); + } + +protected: + kernel_arguments_data get_arguments(const typed_primitive_inst& instance) const override { + kernel_arguments_data args; + for (size_t i = 0; i < instance.inputs_memory_count(); i++) { + args.inputs.push_back(instance.input_memory_ptr(i)); + } + + for (size_t i = 0; i < instance.outputs_memory_count(); i++) { + args.outputs.push_back(instance.output_memory_ptr(i)); + } + return args; + } + +public: + static kernel_params_t get_kernel_params(const kernel_impl_params& impl_param) { + const auto& primitive = impl_param.typed_desc(); + auto params = get_default_params(impl_param); + params.sequential = true; + for (size_t i = 1; i < impl_param.input_layouts.size(); ++i) { + params.inputs.push_back(convert_data_tensor(impl_param.get_input_layout(i))); + } + + if (!primitive->activations.empty()) { + auto a_sz = primitive->activations.size(); + auto param_sz = primitive->activation_params.size(); + OPENVINO_ASSERT(param_sz == 0|| a_sz == param_sz, "[GPU] Unexpected activation params count in lstm_seq impl: ", param_sz); + for (size_t i = 0; i < a_sz; i++) { + params.activations.emplace_back(get_kernel_selector_activation_param(primitive->activations[i]), + param_sz ? primitive->activation_params[i].a : 0.0f, + param_sz ? primitive->activation_params[i].b : 0.0f); + } + } + + if (primitive->clip > 0.0f) { + params.activations.emplace_back(get_kernel_selector_activation_param(activation_func::clamp), -primitive->clip, primitive->clip); + } + + params.SetOffsetOrder(static_cast(primitive->offset_order)); + params.clip = primitive->clip; + params.direction = primitive->direction; + return params; + } + + static kernel_impl_params static_canonicalize_shapes(const kernel_impl_params& impl_params) { + if (impl_params.get_input_layout().get_partial_shape().size() != 3) { + return primitive_impl::static_canonicalize_shapes(impl_params); + } + auto updated_impl_params = canonicalize_fused_shapes(impl_params); + return updated_impl_params; + } + + kernel_impl_params canonicalize_shapes(const kernel_impl_params& impl_params) const override { + return static_canonicalize_shapes(impl_params); + } +}; + +std::unique_ptr RNNSeqImplementationManager::create_impl(const program_node& node, const kernel_impl_params& params) const { + OPENVINO_ASSERT(node.is_type()); + return typed_primitive_impl_ocl::create(static_cast(node), params); +} + +} // namespace ocl +} // namespace cldnn + +BIND_BINARY_BUFFER_WITH_TYPE(cldnn::ocl::rnn_seq_impl) +BIND_BINARY_BUFFER_WITH_TYPE(cldnn::lstm_seq) diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/rnn_seq.hpp b/src/plugins/intel_gpu/src/graph/impls/ocl/rnn_seq.hpp new file mode 100644 index 00000000000000..3e71ad2be51192 --- /dev/null +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/rnn_seq.hpp @@ -0,0 +1,46 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "lstm_seq_inst.h" +#include "impls/registry/implementation_manager.hpp" +#include "intel_gpu/runtime/layout.hpp" + +#include +namespace cldnn { +namespace ocl { + +struct RNNSeqImplementationManager: public ImplementationManager { + OV_GPU_PRIMITIVE_IMPL("ocl::lstm_seq") + RNNSeqImplementationManager(shape_types shape_type, ValidateFunc vf = nullptr) : ImplementationManager(impl_types::ocl, shape_type, vf) {} + + std::unique_ptr create_impl(const program_node& node, const kernel_impl_params& params) const override; + + bool validate_impl(const program_node& node) const override { + assert(node.is_type()); + + const auto& input_layout = node.get_input_layout(0); + const auto& output_layout = node.get_output_layout(0); + + auto input_fmt = input_layout.format; + auto output_fmt = output_layout.format; + auto in_dt = input_layout.data_type; + auto out_dt = output_layout.data_type; + static const std::vector supported_formats = { + format::bfyx + }; + static const std::vector supported_data_types = { + data_types::f32, + data_types::f16, + }; + + if (!one_of(in_dt, supported_data_types) || !one_of(out_dt, supported_data_types)) { + return false; + } + + return one_of(input_fmt.value, supported_formats) && one_of(output_fmt.value, supported_formats); + } +}; + +} // namespace ocl +} // namespace cldnn diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/rope.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl/rope.cpp index 27ce085ab83c3f..d06e643c71ad18 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl/rope.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/rope.cpp @@ -45,7 +45,7 @@ struct rope_impl : typed_primitive_impl_ocl { params.slice_stop = primitive->config.slice_stop; params.axis = primitive->config.is_qwen || primitive->config.is_chatglm ? 2 : 3; - params.num_of_inputs = primitive->config.is_chatglm || primitive->config.is_interleaved ? 2 : 3; + params.num_of_inputs = primitive->config.is_chatglm || (primitive->config.output_trans0213 && primitive->config.is_interleaved) ? 2 : 3; if (params.gather_rank > 0) { params.num_of_inputs++; @@ -53,6 +53,7 @@ struct rope_impl : typed_primitive_impl_ocl { params.is_qwen = primitive->config.is_qwen; params.is_chatglm = primitive->config.is_chatglm; + params.is_interleaved = primitive->config.is_interleaved; params.support_2d_rope = primitive->config.support_2d_rope; params.transposed_input = primitive->config.input_trans0213; diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/search_sorted.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl/search_sorted.cpp new file mode 100644 index 00000000000000..4243d75b5c7367 --- /dev/null +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/search_sorted.cpp @@ -0,0 +1,107 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "primitive_base.hpp" +#include "search_sorted/search_sorted_kernel_base.h" +#include "search_sorted/search_sorted_kernel_selector.h" +#include "search_sorted_inst.h" + +namespace cldnn { +namespace ocl { + +struct search_sorted_impl : typed_primitive_impl_ocl { + using parent = typed_primitive_impl_ocl; + using parent::parent; + using kernel_selector_t = kernel_selector::search_sorted_kernel_selector; + using kernel_params_t = kernel_selector::search_sorted_params; + + DECLARE_OBJECT_TYPE_SERIALIZATION(cldnn::ocl::search_sorted_impl) + + std::unique_ptr clone() const override { + return make_unique(*this); + } + + void load(BinaryInputBuffer& ib) override { + parent::load(ib); + if (is_dynamic()) { + auto& kernel_selector = kernel_selector_t::Instance(); + auto kernel_impl = kernel_selector.GetImplementation(_kernel_data.kernelName); + kernel_impl->GetUpdateDispatchDataFunc(_kernel_data); + } + } + + void update_dispatch_data(const kernel_impl_params& impl_param) override { + // If model loaded from cache, params are not initialized, so we create a new object and reuse it in the future + if (_kernel_data.params == nullptr) { + _kernel_data.params = std::make_shared(get_kernel_params(impl_param, true)); + } + + update_shapes(*_kernel_data.params, impl_param); + (_kernel_data.update_dispatch_data_func)(*_kernel_data.params, _kernel_data); + } + + static kernel_params_t get_kernel_params(const kernel_impl_params& impl_param, bool shape_agnostic = false) { + const auto& primitive = impl_param.typed_desc(); + auto params = get_default_params(impl_param, shape_agnostic); + + // Manually add all inputs except first one, since get_default_params does not handle it. + for (size_t i = 1; i < impl_param.input_layouts.size(); ++i) { + params.inputs.push_back(convert_data_tensor(impl_param.get_input_layout(i))); + } + + params.right_mode = primitive->right_mode; + return params; + } + + // [NOTE]: Has to be added as a separete static function, since it is called via static dispatching in + // typed_primitive_impl_ocl::create().. + static kernel_impl_params static_canonicalize_shapes(const kernel_impl_params& impl_params) { + auto updated_impl_params = canonicalize_fused_shapes(impl_params); + + for (auto& input_layout : updated_impl_params.input_layouts) { + input_layout.set_partial_shape(extend_shape_to_rank_from_begin(input_layout.get_partial_shape())); + } + + for (auto& output_layout : updated_impl_params.output_layouts) { + output_layout.set_partial_shape(extend_shape_to_rank_from_begin(output_layout.get_partial_shape())); + } + + return updated_impl_params; + } + + kernel_impl_params canonicalize_shapes(const kernel_impl_params& impl_params) const override { + return static_canonicalize_shapes(impl_params); + } +}; + +namespace detail { + +attach_search_sorted_impl::attach_search_sorted_impl() { + auto types = { + data_types::i8, + data_types::u8, + data_types::i16, + data_types::u16, + data_types::i32, + data_types::u32, + data_types::i64, + data_types::f16, + data_types::f32, + }; + + auto formats = {format::bfyx, format::bfzyx}; + + implementation_map::add(impl_types::ocl, + shape_types::any, + typed_primitive_impl_ocl::create, + types, + formats); +} + +} // namespace detail +} // namespace ocl +} // namespace cldnn + +BIND_BINARY_BUFFER_WITH_TYPE(cldnn::ocl::search_sorted_impl) +BIND_BINARY_BUFFER_WITH_TYPE(cldnn::search_sorted) diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/lstm_seq_onednn.cpp b/src/plugins/intel_gpu/src/graph/impls/onednn/lstm_seq_onednn.cpp new file mode 100644 index 00000000000000..8189e6e7b37412 --- /dev/null +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/lstm_seq_onednn.cpp @@ -0,0 +1,217 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "lstm_seq_inst.h" +#include "primitive_onednn_base.h" +#include "lstm_seq_onednn.hpp" +#include "impls/registry/implementation_map.hpp" + +#include "kernel_selector_common.h" + +#include + +#include +#include +namespace cldnn { +namespace onednn { + +struct lstm_seq_onednn : typed_primitive_onednn_impl { + using parent = typed_primitive_onednn_impl; + using parent::parent; + + DECLARE_OBJECT_TYPE_SERIALIZATION(cldnn::onednn::lstm_seq_onednn) + +protected: + std::unique_ptr clone() const override { + return make_unique(*this); + } + + std::unordered_map get_arguments(lstm_seq_inst& instance) const override { + std::unordered_map args; + std::vector> dnnl_arg{{DNNL_ARG_SRC_LAYER, DNNL_ARG_SRC_ITER, DNNL_ARG_SRC_ITER_C}, {DNNL_ARG_WEIGHTS_LAYER, + DNNL_ARG_WEIGHTS_ITER, DNNL_ARG_BIAS}, {DNNL_ARG_DST_LAYER, DNNL_ARG_DST_ITER, DNNL_ARG_DST_ITER_C}}; + + for (int i = 0; i < 3; i++) { + for (int j = 0 ; j < 3; j++) { + dnnl::memory mem; + switch (i) { + case 0: + { + auto& input = instance.input_memory(j); + auto offset = onednn::get_offset(instance.get_input_layout(j), _pd.dnnl::primitive_desc_base::src_desc(j)); + mem = input.get_onednn_memory(_pd.dnnl::primitive_desc_base::src_desc(j), offset); + break; + } + case 1: + { + auto& input = instance.input_memory(3+j); + auto offset = onednn::get_offset(instance.get_input_layout(3+j), _pd.dnnl::primitive_desc_base::weights_desc(j)); + mem = input.get_onednn_memory(_pd.dnnl::primitive_desc_base::weights_desc(j), offset); + break; + } + case 2: + { + auto& output = instance.output_memory(j); + auto offset = onednn::get_offset(instance.get_output_layout(j), _pd.dnnl::primitive_desc_base::dst_desc(j)); + mem = output.get_onednn_memory(_pd.dnnl::primitive_desc_base::dst_desc(j), offset); + break; + } + default: + break; + } + args.insert({dnnl_arg[i][j], mem}); + } + } + return args; + } + + static cldnn::layout get_reorder_layout(const kernel_impl_params& impl_params, size_t layout_nr) { + auto weights_shape = impl_params.get_input_layout(layout_nr).get_shape(); + auto target_weights_layout = impl_params.get_input_layout(layout_nr); + target_weights_layout.format = cldnn::format::bfzyx; + auto layout = target_weights_layout.clone_with_other_shape(ov::Shape{weights_shape[0], weights_shape[1], weights_shape[2], 1, 1}); + return layout; + } + + static std::shared_ptr get_weights_reorder(const kernel_impl_params& impl_params, const dnnl::primitive_desc& pd) { + const auto weights_layout_idx = 3; + auto source_weights_layout = impl_params.get_input_layout(weights_layout_idx); + auto target_weights_layout = get_reorder_layout(impl_params, weights_layout_idx); + auto W_desc = onednn::layout_to_memory_desc(source_weights_layout); + auto grouped_weights = format::is_grouped(source_weights_layout.format); + + return std::make_shared(source_weights_layout, + target_weights_layout, + W_desc, + W_desc, + false, + grouped_weights); + } + static std::shared_ptr get_lstm_primitive_descriptor(const kernel_impl_params& impl_params, cldnn::engine& engine, + const dnnl::primitive_attr& attr, + ov::op::RecurrentSequenceDirection direction) { + auto prim = impl_params.typed_desc(); + auto num_dir = static_cast(prim->num_directions()); + const auto& src_shape = impl_params.get_input_layout(0).get_shape(); + auto mod_src_shape = src_shape; + std::swap(mod_src_shape[0], mod_src_shape[1]); + auto input_md = onednn::layout_to_memory_desc(impl_params.get_input_layout(0).clone_with_other_shape(mod_src_shape), dnnl::memory::format_tag::abc); + auto initial_hidden_shape_mod = impl_params.get_input_layout(1).get_shape(); + initial_hidden_shape_mod = { 1, num_dir, initial_hidden_shape_mod[0], initial_hidden_shape_mod[2] }; + auto initial_hidden = onednn::layout_to_memory_desc(impl_params.get_input_layout(1).clone_with_other_shape(initial_hidden_shape_mod)); + auto initial_cell = onednn::layout_to_memory_desc(impl_params.get_input_layout(2).clone_with_other_shape(initial_hidden_shape_mod)); + auto W_shape_mod = impl_params.get_input_layout(3).get_shape(); + W_shape_mod = {1, num_dir, W_shape_mod[2], 4, W_shape_mod[1]/4}; + auto w_layout = impl_params.get_input_layout(3).clone_with_other_shape(W_shape_mod); + w_layout.format = cldnn::format::bfzyx; + auto W_md = onednn::layout_to_memory_desc(w_layout); + auto R_shape_mod = impl_params.get_input_layout(4).get_shape(); + R_shape_mod = {1, num_dir, R_shape_mod[2], 4, R_shape_mod[1]/4}; + auto r_layout = impl_params.get_input_layout(4).clone_with_other_shape(R_shape_mod); + r_layout.format = cldnn::format::bfzyx; + auto R_md = onednn::layout_to_memory_desc(r_layout); + auto B_shape_mod = impl_params.get_input_layout(5).get_shape(); + B_shape_mod = {1, num_dir, 4, B_shape_mod[1]/4}; + auto b_layout = impl_params.get_input_layout(5).clone_with_other_shape(B_shape_mod); + b_layout.format = cldnn::format::bfyx; + auto B_md = onednn::layout_to_memory_desc(b_layout); + auto out_shape = impl_params.get_output_layout().get_shape(); + out_shape = {out_shape[2], out_shape[0], out_shape[3]*num_dir}; + auto output_md = onednn::layout_to_memory_desc(impl_params.get_output_layout().clone_with_other_shape(out_shape), dnnl::memory::format_tag::abc); + auto output1_md = onednn::layout_to_memory_desc(impl_params.get_output_layout(1).clone_with_other_shape(initial_hidden_shape_mod)); + auto output2_md = onednn::layout_to_memory_desc(impl_params.get_output_layout(2).clone_with_other_shape(initial_hidden_shape_mod)); + OPENVINO_ASSERT(input_md.get_format_kind() != dnnl::memory::format_kind::any, + "[GPU] The format kind of the input memory descriptor of onednn lstm_seq cannot be 'any'."); + OPENVINO_ASSERT(output_md.get_format_kind() != dnnl::memory::format_kind::any, + "[GPU] The format kind of the output memory descriptor of onednn lstm_seq cannot be 'any'."); + + auto eng = engine.get_onednn_engine(); + dnnl::rnn_direction lstm_desc_dir; + if (direction == ov::op::RecurrentSequenceDirection::FORWARD) { + lstm_desc_dir = dnnl::rnn_direction::unidirectional_left2right; + } else if (direction == ov::op::RecurrentSequenceDirection::REVERSE) { + lstm_desc_dir = dnnl::rnn_direction::unidirectional_right2left; + } else { + lstm_desc_dir = dnnl::rnn_direction::bidirectional_concat; + } + return std::make_shared( + eng, + dnnl::prop_kind::forward_inference, + lstm_desc_dir, + input_md, + initial_hidden, + initial_cell, + W_md, + R_md, + B_md, + output_md, + output1_md, + output2_md); + } + +public: + void save(BinaryOutputBuffer& ob) const override { +#ifdef ONEDNN_PRIMITIVE_SERIALIZATION + parent::save(ob); + + std::vector prim_cache; + prim_cache = _prim.get_cache_blob(); + ob << prim_cache; +#endif + } + + void load(BinaryInputBuffer& ib) override { +#ifdef ONEDNN_PRIMITIVE_SERIALIZATION + parent::load(ib); + + const kernel_impl_params* impl_params = reinterpret_cast(ib.getKernelImplParams()); + + auto input_md = onednn::layout_to_memory_desc(impl_params->get_input_layout(0)); + auto initial_hidden_md = onednn::layout_to_memory_desc(impl_params->get_input_layout(1)); + auto initial_cell_md = onednn::layout_to_memory_desc(impl_params->get_input_layout(2)); + auto W_md = onednn::layout_to_memory_desc(impl_params->get_input_layout(3)); + auto R_md = onednn::layout_to_memory_desc(impl_params->get_input_layout(4)); + auto B_md = onednn::layout_to_memory_desc(impl_params->get_input_layout(5)); + auto output_md = onednn::layout_to_memory_desc(impl_params->get_output_layout()); + auto output2_md = onednn::layout_to_memory_desc(impl_params->get_output_layout()); + auto prim_desc = std::make_shared( + ib.get_engine().get_onednn_engine(), + dnnl::prop_kind::forward_inference, + dnnl::rnn_direction::undef, + input_md, + initial_hidden_md, + initial_cell_md, + W_md, + R_md, + B_md, + output_md, + output_md, + output2_md); + _pd = *prim_desc; + + std::vector prim_cache; + ib >> prim_cache; + _prim = dnnl::primitive(_pd, prim_cache); +#endif + } + + static std::unique_ptr create(const lstm_seq_node& arg, const kernel_impl_params& impl_params) { + auto& engine = impl_params.prog->get_engine(); + auto& config = impl_params.prog->get_config(); + auto attr = impl_params.attrs_onednn; + auto direction = arg.direction(); + auto prim_desc = get_lstm_primitive_descriptor(impl_params, engine, *attr, direction); + return cldnn::make_unique(engine, config, attr, *prim_desc, get_weights_reorder(impl_params, *prim_desc)); + } +}; + +std::unique_ptr LSTMSeqImplementationManager::create_impl(const program_node& node, const kernel_impl_params& params) const { + assert(node.is_type()); + return onednn::lstm_seq_onednn::create(static_cast(node), params); +} + +} // namespace onednn +} // namespace cldnn + +BIND_BINARY_BUFFER_WITH_TYPE(cldnn::onednn::lstm_seq_onednn) diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/lstm_seq_onednn.hpp b/src/plugins/intel_gpu/src/graph/impls/onednn/lstm_seq_onednn.hpp new file mode 100644 index 00000000000000..6fd16a4dd04acf --- /dev/null +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/lstm_seq_onednn.hpp @@ -0,0 +1,89 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "lstm_seq_inst.h" +#include "reshape_inst.h" +#include "intel_gpu/runtime/utils.hpp" +#include "impls/registry/implementation_manager.hpp" +#include "transformations/utils/utils.hpp" +#include "impls/onednn/utils.hpp" + +#include + + +namespace cldnn { +namespace onednn { + +struct LSTMSeqImplementationManager : public ImplementationManager { + OV_GPU_PRIMITIVE_IMPL("onednn::lstm_seq") + LSTMSeqImplementationManager(shape_types shape_type) : ImplementationManager(impl_types::onednn, shape_type) {} + std::unique_ptr create_impl(const program_node& node, const kernel_impl_params& params) const override; + + bool validate_impl(const program_node& node) const override { + assert(node.is_type()); + const auto& info = node.get_program().get_engine().get_device_info(); + if (info.arch == gpu_arch::unknown) + return false; + + const auto& lstm_seq_node = node.as(); + const auto& in_layout = lstm_seq_node.get_input_layout(0); + const auto& out_layout = lstm_seq_node.get_output_layout(0); + + if (node.get_input_layout(0).format != cldnn::format::bfyx && node.get_input_layout(0).format != cldnn::format::fbyx + && node.get_input_layout(0).format != cldnn::format::ybfx) + return false; + + if (!is_supported_pad(in_layout) || !is_supported_pad(out_layout)) + return false; + + auto in0_dt = node.get_input_layout(0).data_type; + auto in1_dt = node.get_input_layout(1).data_type; + auto in2_dt = node.get_input_layout(2).data_type; + auto in3_dt = node.get_input_layout(3).data_type; + auto in4_dt = node.get_input_layout(4).data_type; + auto in5_dt = node.get_input_layout(5).data_type; + auto out0_dt = node.get_output_layout(0).data_type; + auto out1_dt = node.get_output_layout(1).data_type; + auto out2_dt = node.get_output_layout(2).data_type; + bool cell_state_check = one_of(in2_dt, {data_types::f16, data_types::bf16, data_types::f32}) && + one_of(out2_dt, {data_types::f16, data_types::bf16, data_types::f32}); + bool f16_case = everyone_is(data_types::f16, in0_dt, in1_dt, in3_dt, in4_dt, out0_dt, out1_dt); + bool f32_case = everyone_is(data_types::f32, in0_dt, in1_dt, in3_dt, in4_dt, in5_dt, out0_dt, out1_dt); + bool u8u8u8_case = one_of(out0_dt, {data_types::u8, data_types::f32}) && everyone_is(data_types::i8, in3_dt, in4_dt) && + everyone_is(data_types::u8, in0_dt, in1_dt, out1_dt) && everyone_is(data_types::f32, in2_dt, in5_dt, out2_dt); + bool f32u8f32_case = everyone_is(data_types::u8, in0_dt) && everyone_is(data_types::i8, in3_dt, in4_dt) && + one_of(out0_dt, {data_types::u8, data_types::f32}) && everyone_is(data_types::f32, in1_dt, in5_dt, out1_dt); + bool s8s8s8_case = everyone_is(data_types::i8, in0_dt, in1_dt, out0_dt, out1_dt) && one_of(out0_dt, {data_types::i8, data_types::f32}) && + everyone_is(data_types::f32, in2_dt, in5_dt, out2_dt); + bool f32s8f32_case = everyone_is(data_types::i8, in0_dt, in3_dt, in4_dt) && one_of(out0_dt, {data_types::i8, data_types::f32}) && + everyone_is(data_types::f32, in1_dt, in5_dt, out1_dt); + + if (!cell_state_check) + return false; + return f16_case || f32_case || u8u8u8_case || f32u8f32_case || s8s8s8_case || f32s8f32_case; + } + + in_out_fmts_t query_formats(const program_node& node) const override { + assert(node.is_type()); + std::vector in_fmts(node.get_dependencies().size(), format::any); + std::vector out_fmts(node.get_outputs_count(), format::bfyx); + + size_t out_rank = node.get_output_layout().get_rank(); + for (size_t idx = 0; idx < node.get_dependencies().size(); idx++) { + if (node.get_dependency(idx).is_constant()) + continue; + + auto target_format = format::get_default_format(out_rank); + if (idx == 0) + in_fmts[idx] = format::fbyx; + in_fmts[idx] = target_format; + } + out_fmts[0] = format::ybfx; + + return {in_fmts, out_fmts}; + } +}; + +} // namespace onednn +} // namespace cldnn diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/utils.cpp b/src/plugins/intel_gpu/src/graph/impls/onednn/utils.cpp index a8aa43671ed048..75e087a25fb48f 100644 --- a/src/plugins/intel_gpu/src/graph/impls/onednn/utils.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/utils.cpp @@ -270,6 +270,10 @@ dnnl::memory::desc layout_to_memory_desc(cldnn::layout l, dnnl::memory::format_t } else if (target_fmt == dnnl::memory::format_tag::ab) { dims.push_back(l.batch()); dims.push_back(l.get_tensor().count() / l.batch()); + } else if (target_fmt == dnnl::memory::format_tag::abc) { + dims.push_back(l.batch()); + dims.push_back(l.feature()); + dims.push_back(l.spatial(1)); } else if (target_fmt == dnnl::memory::format_tag::ba) { dims.push_back(l.feature()); dims.push_back(l.get_tensor().count() / l.feature()); diff --git a/src/plugins/intel_gpu/src/graph/impls/registry/lstm_cell_impls.cpp b/src/plugins/intel_gpu/src/graph/impls/registry/lstm_cell_impls.cpp new file mode 100644 index 00000000000000..09ba1f670b29d3 --- /dev/null +++ b/src/plugins/intel_gpu/src/graph/impls/registry/lstm_cell_impls.cpp @@ -0,0 +1,27 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "primitive_inst.h" +#include "registry.hpp" +#include "intel_gpu/primitives/rnn.hpp" + +#if OV_GPU_WITH_OCL + #include "impls/ocl/lstm_cell.hpp" +#endif + +namespace ov { +namespace intel_gpu { + +using namespace cldnn; + +const std::vector>& Registry::get_implementations() { + static const std::vector> impls = { + OV_GPU_CREATE_INSTANCE_OCL(ocl::LSTMCellImplementationManager, shape_types::static_shape) + }; + + return impls; +} + +} // namespace intel_gpu +} // namespace ov diff --git a/src/plugins/intel_gpu/src/graph/impls/registry/lstm_seq_impls.cpp b/src/plugins/intel_gpu/src/graph/impls/registry/lstm_seq_impls.cpp new file mode 100644 index 00000000000000..4b718bd1c74c72 --- /dev/null +++ b/src/plugins/intel_gpu/src/graph/impls/registry/lstm_seq_impls.cpp @@ -0,0 +1,32 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "primitive_inst.h" +#include "registry.hpp" +#include "intel_gpu/primitives/rnn.hpp" + +#if OV_GPU_WITH_OCL + #include "impls/ocl/rnn_seq.hpp" +#endif + +#if OV_GPU_WITH_ONEDNN + #include "impls/onednn/lstm_seq_onednn.hpp" +#endif + +namespace ov { +namespace intel_gpu { + +using namespace cldnn; + +const std::vector>& Registry::get_implementations() { + static const std::vector> impls = { + OV_GPU_CREATE_INSTANCE_ONEDNN(onednn::LSTMSeqImplementationManager, shape_types::static_shape) + OV_GPU_CREATE_INSTANCE_OCL(ocl::RNNSeqImplementationManager, shape_types::static_shape) + }; + + return impls; +} + +} // namespace intel_gpu +} // namespace ov diff --git a/src/plugins/intel_gpu/src/graph/impls/registry/registry.hpp b/src/plugins/intel_gpu/src/graph/impls/registry/registry.hpp index a6bb8ad6eebcc2..b7dbbaef6e64f1 100644 --- a/src/plugins/intel_gpu/src/graph/impls/registry/registry.hpp +++ b/src/plugins/intel_gpu/src/graph/impls/registry/registry.hpp @@ -130,6 +130,8 @@ REGISTER_IMPLS(fully_connected); REGISTER_IMPLS(gather); REGISTER_IMPLS(gather_nd); REGISTER_IMPLS(gemm); +REGISTER_IMPLS(lstm_cell); +REGISTER_IMPLS(lstm_seq); REGISTER_IMPLS(pooling); REGISTER_IMPLS(reduce); REGISTER_IMPLS(reorder); @@ -171,7 +173,6 @@ REGISTER_DEFAULT_IMPLS(grid_sample, OCL_S); REGISTER_DEFAULT_IMPLS(group_normalization, OCL_S, OCL_D); REGISTER_DEFAULT_IMPLS(kv_cache, OCL_S, OCL_D); REGISTER_DEFAULT_IMPLS(lrn, OCL_S); -REGISTER_DEFAULT_IMPLS(lstm_elt, OCL_S); REGISTER_DEFAULT_IMPLS(multiclass_nms, OCL_S); REGISTER_DEFAULT_IMPLS(multinomial, OCL_S); REGISTER_DEFAULT_IMPLS(mutable_data, OCL_S); @@ -214,3 +215,4 @@ REGISTER_DEFAULT_IMPLS(unique_count, OCL_S, OCL_D); REGISTER_DEFAULT_IMPLS(unique_gather, OCL_S, OCL_D); REGISTER_DEFAULT_IMPLS(scaled_dot_product_attention, OCL_S, OCL_D); REGISTER_DEFAULT_IMPLS(rope, OCL_S, OCL_D); +REGISTER_DEFAULT_IMPLS(search_sorted, OCL_S, OCL_D); diff --git a/src/plugins/intel_gpu/src/graph/include/layout_optimizer.h b/src/plugins/intel_gpu/src/graph/include/layout_optimizer.h index 52abc5f0cf8cb4..e7d5bdc8bdabdf 100644 --- a/src/plugins/intel_gpu/src/graph/include/layout_optimizer.h +++ b/src/plugins/intel_gpu/src/graph/include/layout_optimizer.h @@ -95,8 +95,7 @@ class layout_optimizer { b_fs_zyx_fsv32_network, b_fs_yx_fsv16_network, b_fs_zyx_fsv16_network, - bs_fs_yx_bsv16_fsv16_network, - use_onednn_impls + bs_fs_yx_bsv16_fsv16_network }; struct optimization_attributes { @@ -107,7 +106,7 @@ class layout_optimizer { int32_t b_fs_yx_fsv16_network = 0; int32_t b_fs_zyx_fsv16_network = 0; int32_t bs_fs_yx_bsv16_fsv16_network = 0; - int32_t use_onednn_impls = 0; + std::map onednn_impls = {}; }; private: @@ -190,6 +189,33 @@ class layout_optimizer { void set_optimization_attribute(optimization_attributes_type attribute, int32_t val); optimization_attributes get_optimization_attributes() { return _optimization_attributes; } + template + void enable_onednn_for() { + _optimization_attributes.onednn_impls[PT::type_id()] = true; + } + + template + void disable_onednn_for() { + _optimization_attributes.onednn_impls[PT::type_id()] = false; + } + void add_all_onednn_impls_optimization_attribute(); + bool has_all_enabled_onednn_impls_optimization_attribute(); + template + bool is_enabled_onednn_for() { + auto type_id = PT::type_id(); + auto it = _optimization_attributes.onednn_impls.find(type_id); + if (it == _optimization_attributes.onednn_impls.end()) { + return false; + } + + return it->second; + } + void set_value_onednn(primitive_type_id p_type, bool val); + bool contains_onednn_impls_optimization_attribute(const program_node*); + bool is_empty_onednn_impls_optimization_attribute(); + void clear_onednn_impls_optimization_attribute(); + std::map get_all_onednn_impls_optimization_attribute(); + void set_implementation_forcing(const ov::intel_gpu::ImplForcingMap& map); const std::map>& get_implementation_forcing() const; diff --git a/src/plugins/intel_gpu/src/graph/include/lstm_cell_inst.h b/src/plugins/intel_gpu/src/graph/include/lstm_cell_inst.h new file mode 100644 index 00000000000000..38c4232a500eb9 --- /dev/null +++ b/src/plugins/intel_gpu/src/graph/include/lstm_cell_inst.h @@ -0,0 +1,38 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once +#include "intel_gpu/primitives/lstm_cell.hpp" +#include "primitive_inst.h" + +#include + +namespace cldnn { +template <> +struct typed_program_node : public typed_program_node_base { + using parent = typed_program_node_base; + +public: + using parent::parent; +}; + +using lstm_cell_node = typed_program_node; + +template <> +class typed_primitive_inst : public typed_primitive_inst_base { + using parent = typed_primitive_inst_base; + using parent::parent; + +public: + template + static std::vector calc_output_layouts(lstm_cell_node const& node, kernel_impl_params const& impl_param); + static layout calc_output_layout(lstm_cell_node const& node, kernel_impl_params const& impl_param); + static std::string to_string(lstm_cell_node const& node); + +public: + typed_primitive_inst(network& network, lstm_cell_node const& node); +}; + +using lstm_cell_inst = typed_primitive_inst; +} // namespace cldnn diff --git a/src/plugins/intel_gpu/src/graph/include/lstm_elt_inst.h b/src/plugins/intel_gpu/src/graph/include/lstm_elt_inst.h deleted file mode 100644 index 1524598c6f3987..00000000000000 --- a/src/plugins/intel_gpu/src/graph/include/lstm_elt_inst.h +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (C) 2018-2024 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once -#include "intel_gpu/primitives/lstm.hpp" -#include "primitive_inst.h" - -#include - -namespace cldnn { -template <> -struct typed_program_node : public typed_program_node_base { - using parent = typed_program_node_base; - -public: - using parent::parent; - - program_node& input() const { return get_dependency(0); } - program_node& cell() const { return get_dependency(1); } - bool cell_term() const { return !get_primitive()->cell.empty(); } - lstm_weights_order offset_order() const { return get_primitive()->offset_order; } - float clip() const { - float clip_val = get_primitive()->clip; - if (clip_val < 0) - throw std::range_error("Clip value < 0"); - return clip_val; - } - bool input_forget() const { return get_primitive()->input_forget; } - int32_t direction() const { return get_primitive()->direction; } -}; - -using lstm_elt_node = typed_program_node; - -template <> -class typed_primitive_inst : public typed_primitive_inst_base { - using parent = typed_primitive_inst_base; - using parent::parent; - -public: - template - static std::vector calc_output_layouts(lstm_elt_node const& node, kernel_impl_params const& impl_param); - static layout calc_output_layout(lstm_elt_node const& node, kernel_impl_params const& impl_param); - static std::string to_string(lstm_elt_node const& node); - -public: - typed_primitive_inst(network& network, lstm_elt_node const& node); - - memory::ptr cell_memory() const { return dep_memory_ptr(1); } - bool cell_term() const { return !get_typed_desc()->cell.empty(); } - lstm_weights_order offset_order() const { return get_typed_desc()->offset_order; } - float clip() const { - float clip_val = get_typed_desc()->clip; - if (clip_val < 0) - throw std::range_error("Clip value < 0"); - return clip_val; - } - bool input_forget() const { return get_typed_desc()->input_forget; } - uint32_t direction() const { return get_typed_desc()->direction; } -}; - -using lstm_elt_inst = typed_primitive_inst; - -} // namespace cldnn diff --git a/src/plugins/intel_gpu/src/graph/include/lstm_seq_inst.h b/src/plugins/intel_gpu/src/graph/include/lstm_seq_inst.h new file mode 100644 index 00000000000000..33ad7bebac2fbc --- /dev/null +++ b/src/plugins/intel_gpu/src/graph/include/lstm_seq_inst.h @@ -0,0 +1,39 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once +#include "intel_gpu/primitives/rnn.hpp" +#include "primitive_inst.h" + +#include + +namespace cldnn { +template <> +struct typed_program_node : public typed_program_node_base { + using parent = typed_program_node_base; + +public: + using parent::parent; + ov::op::RecurrentSequenceDirection direction() const { return get_primitive()->direction; } +}; + +using lstm_seq_node = typed_program_node; + +template <> +class typed_primitive_inst : public typed_primitive_inst_base { + using parent = typed_primitive_inst_base; + using parent::parent; + +public: + template + static std::vector calc_output_layouts(lstm_seq_node const& node, kernel_impl_params const& impl_param); + static layout calc_output_layout(lstm_seq_node const& node, kernel_impl_params const& impl_param); + static std::string to_string(lstm_seq_node const& node); + +public: + typed_primitive_inst(network& network, lstm_seq_node const& node); +}; + +using lstm_seq_inst = typed_primitive_inst; +} // namespace cldnn diff --git a/src/plugins/intel_gpu/src/graph/include/pass_manager.h b/src/plugins/intel_gpu/src/graph/include/pass_manager.h index 281f95a892662c..61c34c0eff548f 100644 --- a/src/plugins/intel_gpu/src/graph/include/pass_manager.h +++ b/src/plugins/intel_gpu/src/graph/include/pass_manager.h @@ -196,6 +196,11 @@ class post_optimize_weights : public base_pass { weights_bias_offset get_weights_bias_offset(const T& node); template void optimize_weights(T& node, program& p); + void select_implementation(program& p, program_node& node); + void add_lstm_weights_reorder(primitive_id input_id, std::shared_ptr reorder_params, program& p, cldnn::program_node&, \ + cldnn::program_node&, size_t); + void add_lstm_bias_reorder(primitive_id input_id, std::shared_ptr reorder_params, program& p, cldnn::program_node&, \ + cldnn::program_node&); reorder_factory& _rf; }; @@ -302,13 +307,8 @@ class memory_dependency_pass : public base_pass { if ((node->can_be_optimized() && !node->is_runtime_skippable()) || !dep->can_be_optimized()) { node->add_memory_dependency(static_cast(dep->get_unique_id())); } else { - if (node->is_runtime_skippable() || dep->is_runtime_skippable()) { + if (node->is_runtime_skippable() || dep->is_runtime_skippable() || dep->can_be_optimized()) { node->add_memory_dependency(static_cast(dep->get_unique_id())); - for (const auto& subdep : dep->get_dependencies()) { - add_memory_dependency(node, subdep.first); - add_memory_dependency(subdep.first, node); - } - return; } for (const auto& subdep : dep->get_dependencies()) { diff --git a/src/plugins/intel_gpu/src/graph/include/primitive_type_base.h b/src/plugins/intel_gpu/src/graph/include/primitive_type_base.h index eb3f313af00de6..81f614ba9fd43a 100644 --- a/src/plugins/intel_gpu/src/graph/include/primitive_type_base.h +++ b/src/plugins/intel_gpu/src/graph/include/primitive_type_base.h @@ -51,7 +51,7 @@ struct primitive_type_base : primitive_type { if ((node.get_forced_impl_type() & impl_type) != impl_type) continue; - if (impl_type == impl_types::onednn && !node.get_program().get_layout_optimizer().get_optimization_attributes().use_onednn_impls) + if (impl_type == impl_types::onednn && !node.get_program().get_layout_optimizer().contains_onednn_impls_optimization_attribute(&node)) continue; shape_types supported_shape_type = impl->get_shape_type(); @@ -168,7 +168,7 @@ struct primitive_type_base : primitive_type { return true; continue; } else { - if (impl_type == impl_types::onednn && !node.get_program().get_layout_optimizer().get_optimization_attributes().use_onednn_impls) + if (impl_type == impl_types::onednn && !node.get_program().get_layout_optimizer().contains_onednn_impls_optimization_attribute(&node)) continue; if (!impl->validate(node)) diff --git a/src/plugins/intel_gpu/src/graph/include/search_sorted_inst.h b/src/plugins/intel_gpu/src/graph/include/search_sorted_inst.h new file mode 100644 index 00000000000000..50ffdf8112e2ae --- /dev/null +++ b/src/plugins/intel_gpu/src/graph/include/search_sorted_inst.h @@ -0,0 +1,46 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#pragma once + +#include + +#include "primitive_inst.h" + +namespace cldnn { + +template <> +struct typed_program_node : public typed_program_node_base { + using parent = typed_program_node_base; + typed_program_node(const std::shared_ptr prim, program& prog) : parent(prim, prog) {} + +public: + using parent::parent; + + program_node& input(size_t idx = 0) const { + return get_dependency(idx); + } + std::vector get_shape_infer_dependencies() const override { + return {}; + } +}; + +using search_sorted_node = typed_program_node; + +template <> +class typed_primitive_inst : public typed_primitive_inst_base { + using parent = typed_primitive_inst_base; + using parent::parent; + +public: + typed_primitive_inst(network& network, search_sorted_node const& desc); + template + static std::vector calc_output_layouts(search_sorted_node const& node, + kernel_impl_params const& impl_param); + static layout calc_output_layout(search_sorted_node const& node, kernel_impl_params const& impl_param); + static std::string to_string(search_sorted_node const& node); +}; + +using search_sorted_inst = typed_primitive_inst; + +} // namespace cldnn diff --git a/src/plugins/intel_gpu/src/graph/layout_optimizer.cpp b/src/plugins/intel_gpu/src/graph/layout_optimizer.cpp index bb4d739b3a07c1..5262e8c4621e72 100644 --- a/src/plugins/intel_gpu/src/graph/layout_optimizer.cpp +++ b/src/plugins/intel_gpu/src/graph/layout_optimizer.cpp @@ -27,7 +27,6 @@ #include "pooling_inst.h" #include "reduce_inst.h" #include "one_hot_inst.h" -#include "permute_inst.h" #include "quantize_inst.h" #include "mvn_inst.h" #include "depth_to_space_inst.h" @@ -37,7 +36,10 @@ #include "gather_inst.h" #include "broadcast_inst.h" #include "loop_inst.h" +#include "concatenation_inst.h" +#include "permute_inst.h" #include "dft_inst.h" +#include "lstm_seq_inst.h" #include "to_string_utils.h" #include #include @@ -114,7 +116,6 @@ bool layout_optimizer::is_format_supported(program_node& node, format::type fmt) node.get_input_layout(0).data_type != data_types::i8 && node.get_input_layout(0).data_type != data_types::u8) return false; - if (node.is_type()) return node.get_output_layout().format == fmt; @@ -132,7 +133,7 @@ bool layout_optimizer::can_fuse_reorder(program_node& prev, program_node& next, auto next_output_layout = next.get_output_layout(); auto prev_dt = prev.get_output_layout().data_type; auto next_dt = next.get_output_layout().data_type; - auto use_onednn_impls = _optimization_attributes.use_onednn_impls; + auto use_onednn_impls = has_all_enabled_onednn_impls_optimization_attribute(); if (prev.is_dynamic() || next.is_dynamic()) return false; @@ -365,7 +366,7 @@ bool layout_optimizer::can_fuse_reorder_to_prev(program_node& prev, reorder_node auto next = node.get_users().front(); auto dt_prev = prev.get_output_layout().data_type; auto dt_next = next->get_output_layout().data_type; - auto use_onednn_impls = _optimization_attributes.use_onednn_impls; + auto use_onednn_impls = contains_onednn_impls_optimization_attribute(&node) && contains_onednn_impls_optimization_attribute(&prev); if (prev.is_type()) return true; @@ -927,7 +928,7 @@ format layout_optimizer::get_expected_format(convolution_node const& node) { } bool onednn_valid_post_ops = get_post_ops_count(node) <= 32; - bool use_onednn_impls = _optimization_attributes.use_onednn_impls && input_layout.data_type != data_types::f32; + bool use_onednn_impls = contains_onednn_impls_optimization_attribute(&node) && input_layout.data_type != data_types::f32; // Use planar bfyx format for dynamic convolutions with explicit padding in clDNN if (node.is_dynamic() && output_layout.get_partial_shape().size() == 4 && node.use_explicit_padding() && !i8_u8_input && @@ -1038,7 +1039,7 @@ format layout_optimizer::get_expected_format(deconvolution_node const& node) { } auto expected_shape = output_layout.get_shape(); - bool use_onednn_impls = _optimization_attributes.use_onednn_impls; + bool use_onednn_impls = contains_onednn_impls_optimization_attribute(&node); auto available = node.get_primitive()->type->get_available_impl_types(node); @@ -1086,7 +1087,7 @@ format layout_optimizer::get_expected_format(quantize_node const& node) { return all_users_gemm; }; - auto use_onednn_impls = _optimization_attributes.use_onednn_impls; + auto use_onednn_impls = has_all_enabled_onednn_impls_optimization_attribute(); if (use_onednn_impls) { expected = format::any; @@ -1126,7 +1127,7 @@ format layout_optimizer::get_expected_format(quantize_node const& node) { bool layout_optimizer::is_primitive_implemented_for_onednn(program_node& node) { if (node.is_type() || node.is_type() || node.is_type() || node.is_type() || node.is_type() || - node.is_type() || node.is_type() || node.is_type()) { + node.is_type() || node.is_type() || node.is_type() || node.is_type()) { return true; } @@ -1219,7 +1220,7 @@ impl_types layout_optimizer::get_preferred_impl_type(program_node& node, format format layout_optimizer::get_preferred_format(program_node& node) { format expected = format::any; auto output_layout = node.get_output_layout(); - bool use_onednn_impls = _optimization_attributes.use_onednn_impls; + bool use_onednn_impls = contains_onednn_impls_optimization_attribute(&node); bool allow_new_shape_infer = node.get_program().is_new_shape_infer(); @@ -1417,14 +1418,55 @@ void layout_optimizer::set_optimization_attribute(optimization_attributes_type a case optimization_attributes_type::bs_fs_yx_bsv16_fsv16_network: _optimization_attributes.bs_fs_yx_bsv16_fsv16_network = val; break; - case optimization_attributes_type::use_onednn_impls: - _optimization_attributes.use_onednn_impls = val; - break; default: throw std::out_of_range("unsupported layout optimization attribute"); } } +void layout_optimizer::add_all_onednn_impls_optimization_attribute() { + enable_onednn_for(); + enable_onednn_for(); + enable_onednn_for(); + enable_onednn_for(); + enable_onednn_for(); + enable_onednn_for(); + enable_onednn_for(); + enable_onednn_for(); + enable_onednn_for(); +} + +bool layout_optimizer::has_all_enabled_onednn_impls_optimization_attribute() { + return is_enabled_onednn_for() && is_enabled_onednn_for() && is_enabled_onednn_for() && + is_enabled_onednn_for() && is_enabled_onednn_for() && is_enabled_onednn_for() && + is_enabled_onednn_for() && is_enabled_onednn_for() && is_enabled_onednn_for(); +} + +void layout_optimizer::set_value_onednn(primitive_type_id p_type, bool val) { + _optimization_attributes.onednn_impls[p_type] = val; +} + +bool layout_optimizer::contains_onednn_impls_optimization_attribute(const program_node* node) { + auto type_id = node->type(); + auto it = _optimization_attributes.onednn_impls.find(type_id); + if (it == _optimization_attributes.onednn_impls.end()) { + return false; + } + + return it->second; +} + +bool layout_optimizer::is_empty_onednn_impls_optimization_attribute() { + return _optimization_attributes.onednn_impls.empty(); +} + +void layout_optimizer::clear_onednn_impls_optimization_attribute() { + _optimization_attributes.onednn_impls.clear(); +} + +std::map layout_optimizer::get_all_onednn_impls_optimization_attribute() { + return _optimization_attributes.onednn_impls; +} + bool layout_optimizer::is_format_optimized(const convolution_node& node, const format& format, bool use_weak_restrictions) { auto input_layout = node.get_input_layout(); auto weights_layout = node.weights().get_output_layout(); diff --git a/src/plugins/intel_gpu/src/graph/lstm_cell.cpp b/src/plugins/intel_gpu/src/graph/lstm_cell.cpp new file mode 100644 index 00000000000000..0b300199fb05a3 --- /dev/null +++ b/src/plugins/intel_gpu/src/graph/lstm_cell.cpp @@ -0,0 +1,51 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#include "lstm_cell_inst.h" +#include "primitive_type_base.h" +#include "json_object.h" +#include + +namespace cldnn { +GPU_DEFINE_PRIMITIVE_TYPE_ID(lstm_cell) + +layout lstm_cell_inst::calc_output_layout(lstm_cell_node const& node, kernel_impl_params const& impl_param) { + const auto& input_layout = impl_param.get_input_layout(0); + const auto& input_pshape = input_layout.get_partial_shape(); + const auto& input_layout_hidden = impl_param.get_input_layout(1); + const auto& input_pshape_hidden = input_layout_hidden.get_partial_shape(); + const auto& lstm_batch_size = input_pshape[0]; + const auto& lstm_hidden_size = input_pshape_hidden[1]; + + return cldnn::layout{ov::PartialShape{lstm_batch_size, lstm_hidden_size}, input_layout.data_type, input_layout.format}; +} + +template +std::vector lstm_cell_inst::calc_output_layouts(lstm_cell_node const& node, kernel_impl_params const& impl_param) { + const auto& input_layout = impl_param.get_input_layout(0); + const auto& input_pshape = input_layout.get_partial_shape(); + const auto& input_layout_hidden = impl_param.get_input_layout(1); + const auto& input_pshape_hidden = input_layout_hidden.get_partial_shape(); + const auto& lstm_batch_size = input_pshape[0]; + const auto& lstm_hidden_size = input_pshape_hidden[1]; + + auto out_layout = cldnn::layout{ShapeType{lstm_batch_size, lstm_hidden_size}, input_layout.data_type, input_layout.format}; + return {out_layout, out_layout}; +} + +template std::vector lstm_cell_inst::calc_output_layouts(lstm_cell_node const& node, const kernel_impl_params& impl_param); + +std::string lstm_cell_inst::to_string(lstm_cell_node const& node) { + auto node_info = node.desc_to_json(); + + std::stringstream primitive_description; + + json_composite lstm_cell_info; + node_info->add("lstm cell info", lstm_cell_info); + node_info->dump(primitive_description); + + return primitive_description.str(); +} + +lstm_cell_inst::typed_primitive_inst(network& network, lstm_cell_node const& node) : parent(network, node) {} +} // namespace cldnn diff --git a/src/plugins/intel_gpu/src/graph/lstm_elt.cpp b/src/plugins/intel_gpu/src/graph/lstm_elt.cpp deleted file mode 100644 index 098e89aa45003e..00000000000000 --- a/src/plugins/intel_gpu/src/graph/lstm_elt.cpp +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright (C) 2018-2024 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// -#include "lstm_elt_inst.h" -#include "primitive_type_base.h" -#include "intel_gpu/runtime/error_handler.hpp" -#include "json_object.h" -#include - -namespace cldnn { -GPU_DEFINE_PRIMITIVE_TYPE_ID(lstm_elt) - -layout lstm_elt_inst::calc_output_layout(lstm_elt_node const& node, kernel_impl_params const& impl_param) { - assert(static_cast(impl_param.desc->output_data_types[0]) == false && - "Output data type forcing is not supported for lstm_elt_node!"); - auto input_layout = impl_param.get_input_layout(); - - // tempGEMM{bfyx} = [b: batch, f: direction, x: 1, y: 4 * hidden_size ] input - // cell{bfyx} = [b: batch, f: direction, x: 1, y: hidden_size ] optional - // output{bfyx} = [b: batch, f: 2, x: direction, y: hidden_size ] output - // The output of the lstm_elt node is the concatenation of the intermediate [hidden, cell] tensors. - // A crop/split node is needed to extract each individual tensors - auto result = - layout(input_layout.data_type, - input_layout.format, - tensor(input_layout.batch(), 2, input_layout.spatial(0) / 4, input_layout.feature())); - return result; -} - -template -std::vector lstm_elt_inst::calc_output_layouts(lstm_elt_node const& node, kernel_impl_params const& impl_param) { - std::vector output_layouts; - - // input partial shape [batch, input_size (= hidden_size * 4)] - auto input_layout = impl_param.get_input_layout(); - auto input_pshape = input_layout.get_partial_shape(); - OPENVINO_ASSERT(static_cast(impl_param.desc->output_data_types[0]) == false, "Output data type forcing is not supported for lstm_elt_node!"); - OPENVINO_ASSERT(input_pshape.rank().get_length() == 2, "input_layout rank should be 2 on dynamic shape."); - - int lstm_input_size, lstm_batch_size, lstm_hidden_size; - if (input_pshape[input_pshape.size() - 1].is_static()) { - lstm_input_size = input_pshape[input_pshape.size() - 1].get_length(); - lstm_hidden_size = lstm_input_size / 4; - } else { - lstm_input_size = -1; - lstm_hidden_size = -1; - } - - if (input_pshape[input_pshape.size() - 2].is_static()) { - lstm_batch_size = input_pshape[input_pshape.size() - 2].get_length(); - } else { - lstm_batch_size = -1; - } - - return {cldnn::layout{ov::PartialShape{lstm_batch_size, 2, 1, lstm_hidden_size}, input_layout.data_type, input_layout.format}}; -} - -template std::vector lstm_elt_inst::calc_output_layouts(lstm_elt_node const& node, const kernel_impl_params& impl_param); - -std::string lstm_elt_inst::to_string(lstm_elt_node const& node) { - auto desc = node.get_primitive(); - auto node_info = node.desc_to_json(); - auto cell_id = desc->cell; - - std::stringstream primitive_description; - - json_composite lstm_elt_info; - lstm_elt_info.add("cell id", cell_id); - node_info->add("lstm elt info", lstm_elt_info); - node_info->dump(primitive_description); - - return primitive_description.str(); -} - -lstm_elt_inst::typed_primitive_inst(network& network, lstm_elt_node const& node) : parent(network, node) { - auto input_size = node.get_input_layout(); - CLDNN_ERROR_NOT_PROPER_FORMAT(node.id(), - "input format", - input_size.format.value, - "expected format", - format::bfyx, - format::fyxb); -} -} // namespace cldnn diff --git a/src/plugins/intel_gpu/src/graph/lstm_seq.cpp b/src/plugins/intel_gpu/src/graph/lstm_seq.cpp new file mode 100644 index 00000000000000..55e25672300cfd --- /dev/null +++ b/src/plugins/intel_gpu/src/graph/lstm_seq.cpp @@ -0,0 +1,71 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#include "lstm_seq_inst.h" +#include "primitive_type_base.h" +#include "json_object.h" +#include + +namespace cldnn { +GPU_DEFINE_PRIMITIVE_TYPE_ID(lstm_seq) + +layout lstm_seq_inst::calc_output_layout(lstm_seq_node const& node, kernel_impl_params const& impl_param) { + const auto& desc = impl_param.typed_desc(); + const auto& input_layout = impl_param.get_input_layout(0); + const auto& input_pshape = input_layout.get_partial_shape(); + const auto& input_layout_hidden = impl_param.get_input_layout(1); + const auto& input_pshape_hidden = input_layout_hidden.get_partial_shape(); + const auto& lstm_batch_size = input_pshape[0]; + const auto& lstm_seq_length = input_pshape[1]; + const auto& lstm_hidden_size = input_pshape_hidden[2]; + + auto first_out_fmt = cldnn::format::bfyx; + if (node.get_preferred_impl_type() == impl_types::onednn && node.get_preferred_output_fmt() != format::any) { + first_out_fmt = node.get_preferred_output_fmt(); + } + + return cldnn::layout{ov::PartialShape{lstm_batch_size, desc->num_directions(), lstm_seq_length, lstm_hidden_size}, input_layout.data_type, first_out_fmt}; +} + +template +std::vector lstm_seq_inst::calc_output_layouts(lstm_seq_node const& node, kernel_impl_params const& impl_param) { + const auto& desc = impl_param.typed_desc(); + const auto& input_layout = impl_param.get_input_layout(0); + const auto& input_pshape = input_layout.get_partial_shape(); + const auto& input_layout_hidden = impl_param.get_input_layout(1); + const auto& input_pshape_hidden = input_layout_hidden.get_partial_shape(); + const auto& lstm_batch_size = input_pshape[0]; + const auto& lstm_seq_length = input_pshape[1]; + const auto& lstm_hidden_size = input_pshape_hidden[2]; + + auto first_out_fmt = cldnn::format::bfyx; + auto second_out_fmt = input_layout.format; + auto third_out_fmt = input_layout.format; + if (node.get_preferred_impl_type() == impl_types::onednn && node.get_preferred_output_fmt() != format::any) { + first_out_fmt = node.get_preferred_output_fmt(); + second_out_fmt = node.get_preferred_output_fmt(1); + third_out_fmt = node.get_preferred_output_fmt(2); + } + auto num_directions = desc->num_directions(); + + return {cldnn::layout{ShapeType{lstm_batch_size, num_directions, lstm_seq_length, lstm_hidden_size}, input_layout.data_type, first_out_fmt}, + cldnn::layout{ShapeType{lstm_batch_size, num_directions, lstm_hidden_size}, input_layout.data_type, second_out_fmt}, + cldnn::layout{ShapeType{lstm_batch_size, num_directions, lstm_hidden_size}, input_layout.data_type, third_out_fmt}}; +} + +template std::vector lstm_seq_inst::calc_output_layouts(lstm_seq_node const& node, const kernel_impl_params& impl_param); + +std::string lstm_seq_inst::to_string(lstm_seq_node const& node) { + auto node_info = node.desc_to_json(); + + std::stringstream primitive_description; + + json_composite lstm_seq_info; + node_info->add("lstm seq info", lstm_seq_info); + node_info->dump(primitive_description); + + return primitive_description.str(); +} + +lstm_seq_inst::typed_primitive_inst(network& network, lstm_seq_node const& node) : parent(network, node) {} +} // namespace cldnn diff --git a/src/plugins/intel_gpu/src/graph/primitive_inst.cpp b/src/plugins/intel_gpu/src/graph/primitive_inst.cpp index dac2c9a3403468..b51c7825b5a8fa 100644 --- a/src/plugins/intel_gpu/src/graph/primitive_inst.cpp +++ b/src/plugins/intel_gpu/src/graph/primitive_inst.cpp @@ -2028,7 +2028,7 @@ primitive_inst::primitive_inst(network & network, program_node const& node, bool , _inputs_memory_count(node.get_inputs_count()) , _outputs_memory_count(node.get_outputs_count()) , _fused_mem_count(node.get_fused_inputs_count()) - , _fused_mem_offset((_fused_mem_count > 0 && node.get_first_fused_dep_idx() > 0) ? node.get_first_fused_dep_idx() : 0) + , _fused_mem_offset((_fused_mem_count > 0 && node.get_first_fused_dep_idx() > 0) ? static_cast(node.get_first_fused_dep_idx()) : 0) , _can_be_optimized(node.can_be_optimized()) , _can_share_buffer(node.can_share_buffer()) , _is_constant(node.is_constant()) @@ -2561,7 +2561,8 @@ cldnn::network::ptr primitive_inst::get_unfused_subgraph() { ExecutionConfig subgraph_config{ ov::intel_gpu::allow_static_input_reorder(true), ov::intel_gpu::allow_new_shape_infer(true), - ov::enable_profiling(get_network().get_config().get_property(ov::enable_profiling)) + ov::enable_profiling(get_network().get_config().get_property(ov::enable_profiling)), + ov::intel_gpu::use_onednn(get_network().get_config().get_property(ov::intel_gpu::use_onednn)) }; auto prog = program::build_program(get_network().get_engine(), t, diff --git a/src/plugins/intel_gpu/src/graph/program.cpp b/src/plugins/intel_gpu/src/graph/program.cpp index 2bfaac84134387..bdffb9c4980722 100644 --- a/src/plugins/intel_gpu/src/graph/program.cpp +++ b/src/plugins/intel_gpu/src/graph/program.cpp @@ -70,6 +70,9 @@ #include "unique_inst.hpp" #include "condition_inst.h" #include "to_string_utils.h" +#include "intel_gpu/graph/serialization/map_serializer.hpp" + +#include "intel_gpu/primitives/rnn.hpp" // TODO: Remove once we have interface for kernels cache #include "impls/ocl/kernels_cache.hpp" @@ -151,15 +154,14 @@ program::program(engine& engine_ref, is_internal(is_internal), _is_body_program(is_body_program), _compilation_context(compilation_context) { - _config.apply_user_properties(_engine.get_device_info()); init_primitives(); GPU_DEBUG_INFO << "Program config\n" << _config.to_string(); init_program(); prepare_nodes(topology); program_node::reset_unique_id(); - if (no_optimizations) { init_graph(); + _config.apply_user_properties(_engine.get_device_info()); } else { build_program(is_internal); if (_is_body_program) { @@ -494,6 +496,7 @@ void program::set_options() { void program::build_program(bool is_internal) { init_graph(); + _config.apply_user_properties(_engine.get_device_info()); { pre_optimize_graph(is_internal); } run_graph_compilation(); { post_optimize_graph(is_internal); } @@ -523,6 +526,9 @@ void program::init_graph() { for (auto& node : processing_order) { if (!node->is_type()) node->get_output_layouts(); + if (node->is_type()) { + _config.set_property(ov::intel_gpu::use_onednn(true)); + } } // Perform initial shape_of subgraphs markup apply_opt_pass(); @@ -1631,11 +1637,17 @@ void program::set_layout_optimizer_attributes(layout_optimizer& lo) { #ifdef ENABLE_ONEDNN_FOR_GPU bool enable_onednn_for_tests = get_config().get_property(ov::intel_gpu::optimize_data) || is_internal_program(); auto& engine = get_engine(); - if (engine.get_device_info().supports_immad && - engine.get_device_info().vendor_id == INTEL_VENDOR_ID && + if (engine.get_device_info().vendor_id == INTEL_VENDOR_ID && get_config().get_property(ov::intel_gpu::queue_type) == QueueTypes::in_order && - enable_onednn_for_tests) - lo.set_optimization_attribute(layout_optimizer::optimization_attributes_type::use_onednn_impls, 1); + enable_onednn_for_tests) { + if (engine.get_device_info().supports_immad) { + lo.add_all_onednn_impls_optimization_attribute(); + } else { + if (get_config().get_property(ov::intel_gpu::use_onednn)) { + lo.enable_onednn_for(); + } + } + } #endif } @@ -1779,7 +1791,13 @@ void program::save(cldnn::BinaryOutputBuffer& ob) const { ob << _is_body_program; ob << _can_be_optimized; - ob << get_layout_optimizer().get_optimization_attributes().use_onednn_impls; + auto onednn_impls_size = get_layout_optimizer().get_all_onednn_impls_optimization_attribute().size(); + ob << onednn_impls_size; + for (const auto& onednn_impl : get_layout_optimizer().get_all_onednn_impls_optimization_attribute()) { + ob << prim_map_storage::instance().get_type_string(onednn_impl.first); + ob << onednn_impl.second; + } + processing_order.save(ob); { @@ -1903,9 +1921,18 @@ void program::load(cldnn::BinaryInputBuffer& ib) { ib >> _is_body_program; ib >> _can_be_optimized; - int32_t use_onednn_attr = 0; - ib >> use_onednn_attr; - get_layout_optimizer().set_optimization_attribute(layout_optimizer::optimization_attributes_type::use_onednn_impls, use_onednn_attr); + + size_t num_of_onednn_impls; + ib >> num_of_onednn_impls; + for (size_t num = 0; num < num_of_onednn_impls; num++) { + primitive_id p_id{}; + bool enabled; + ib >> p_id; + ib >> enabled; + auto ptype_id = prim_map_storage::instance().get_type_id(p_id); + get_layout_optimizer().set_value_onednn(ptype_id, enabled); + } + _loaded_from_cache = true; processing_order.load(ib, *this); diff --git a/src/plugins/intel_gpu/src/graph/program_node.cpp b/src/plugins/intel_gpu/src/graph/program_node.cpp index a9bb7c665f177b..201fa3a155caa9 100644 --- a/src/plugins/intel_gpu/src/graph/program_node.cpp +++ b/src/plugins/intel_gpu/src/graph/program_node.cpp @@ -1505,7 +1505,7 @@ void program_node::create_onednn_primitive_attributes( oc_dim = static_cast(desc.output_layout.get_partial_shape()[1].get_max_length()); else oc_dim = static_cast(desc.output_layout.get_tensor().feature.size()); - post_ops.append_prelu(1 << std::max(0, oc_dim)); + post_ops.append_prelu(1 << static_cast(std::max(0, oc_dim))); update_onednn_post_op_list(onednn_post_op_type::binary_relu, dep_idx); } else if (fused_desc->activation_function == cldnn::activation_func::hard_sigmoid) { post_ops.append_eltwise(dnnl::algorithm::eltwise_hardsigmoid, fused_desc->additional_params.a, fused_desc->additional_params.b); diff --git a/src/plugins/intel_gpu/src/graph/rope.cpp b/src/plugins/intel_gpu/src/graph/rope.cpp index e168626f8d69a2..bef3f6dfcd93c0 100644 --- a/src/plugins/intel_gpu/src/graph/rope.cpp +++ b/src/plugins/intel_gpu/src/graph/rope.cpp @@ -54,7 +54,7 @@ std::vector rope_inst::calc_output_layouts(rope_node const& node, kernel output_shape[3] = input_slice_size; } - if (desc->config.input_trans0213 || desc->config.is_interleaved) { + if (desc->config.input_trans0213 || desc->config.output_trans0213) { std::swap(output_shape[2], output_shape[1]); } } @@ -77,6 +77,7 @@ std::string rope_inst::to_string(rope_node const& node) { rope_info.add("input_trans0213", desc->config.input_trans0213); rope_info.add("is_chatglm", desc->config.is_chatglm); rope_info.add("support_2d_rope", desc->config.support_2d_rope); + rope_info.add("output_trans0213", desc->config.output_trans0213); rope_info.add("is_interleaved", desc->config.is_interleaved); rope_info.add("is_qwen", desc->config.is_qwen); rope_info.add("rotary_ndims", desc->config.rotary_ndims); diff --git a/src/plugins/intel_gpu/src/graph/search_sorted.cpp b/src/plugins/intel_gpu/src/graph/search_sorted.cpp new file mode 100644 index 00000000000000..761b6751ace3b7 --- /dev/null +++ b/src/plugins/intel_gpu/src/graph/search_sorted.cpp @@ -0,0 +1,59 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include + +#include + +#include "openvino/core/enum_names.hpp" +#include "primitive_type_base.h" +#include "search_sorted_shape_inference.hpp" + +namespace cldnn { +GPU_DEFINE_PRIMITIVE_TYPE_ID(search_sorted) + +search_sorted_inst::typed_primitive_inst(network& network, search_sorted_node const& node) : parent(network, node) {} + +layout search_sorted_inst::calc_output_layout(search_sorted_node const& node, kernel_impl_params const& impl_param) { + return calc_output_layouts(node, impl_param)[0]; +} + +template +std::vector search_sorted_inst::calc_output_layouts(search_sorted_node const& node, + kernel_impl_params const& impl_param) { + auto primitive = impl_param.typed_desc(); + + auto input0_layout = impl_param.get_input_layout(0); + auto input1_layout = impl_param.get_input_layout(1); + + const data_types output_type = impl_param.desc->output_data_types[0].value_or(data_types::i64); + + std::vector input_shapes = { + input0_layout.get(), // sorted shape + input1_layout.get(), // values shape + }; + + std::vector output_shapes; + + ov::op::v15::SearchSorted op; + op.set_right_mode(primitive->right_mode); + output_shapes = shape_infer(&op, input_shapes); + + return {layout{output_shapes[0], output_type, input1_layout.format}}; +} + +std::string search_sorted_inst::to_string(search_sorted_node const& node) { + auto node_info = node.desc_to_json(); + json_composite search_sorted_info; + search_sorted_info.add("sorted id", node.input(0).id()); + search_sorted_info.add("values id", node.input(1).id()); + search_sorted_info.add("right_mode", node.get_primitive()->right_mode); + node_info->add("search_sorted info", search_sorted_info); + std::stringstream primitive_description; + node_info->dump(primitive_description); + return primitive_description.str(); +} + +} // namespace cldnn \ No newline at end of file diff --git a/src/plugins/intel_gpu/src/graph/strided_slice.cpp b/src/plugins/intel_gpu/src/graph/strided_slice.cpp index 9c33ec22b0b7c1..47248cd2a4d773 100644 --- a/src/plugins/intel_gpu/src/graph/strided_slice.cpp +++ b/src/plugins/intel_gpu/src/graph/strided_slice.cpp @@ -193,6 +193,10 @@ void strided_slice_inst::on_execute() { void strided_slice_inst::update_output_memory() { if (!can_be_optimized()) return; + + if (node->get_program().is_new_shape_infer() && input_memory_ptr() == nullptr) + return; + if (static_cast(_outputs[0]) && _network.get_engine().is_the_same_buffer(output_memory(), input_memory())) return; diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/lstm_cell_and_seq_bfyx.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/lstm_cell_and_seq_bfyx.cl new file mode 100644 index 00000000000000..f2cf2ca985e855 --- /dev/null +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/lstm_cell_and_seq_bfyx.cl @@ -0,0 +1,215 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "include/batch_headers/fetch_data.cl" +#include "include/batch_headers/common.cl" + +#define INPUT0_TYPE_VEC MAKE_VECTOR_TYPE(INPUT0_TYPE, VEC_SIZE) +#define INPUT1_TYPE_VEC MAKE_VECTOR_TYPE(INPUT1_TYPE, VEC_SIZE) +#define INPUT3_TYPE_VEC MAKE_VECTOR_TYPE(INPUT3_TYPE, VEC_SIZE) +#define INPUT4_TYPE_VEC MAKE_VECTOR_TYPE(INPUT4_TYPE, VEC_SIZE) +#define OUTPUT_TYPE_VEC MAKE_VECTOR_TYPE(OUTPUT_TYPE, VEC_SIZE) +#define READ_VEC(offset, ptr) CAT(vload, VEC_SIZE)(offset, ptr) + +#ifdef SEQUENCE +#define GET_IN0_IDX(b, f, y) INPUT0_GET_INDEX(b, f, y, 0) + #if DIRECTION == 2 + #define GET_IN1_IDX(b, f, y) INPUT1_GET_INDEX(b, f, y, 0) + #define GET_IN2_IDX(b, f, y) INPUT2_GET_INDEX(b, f, y, 0) + #define GET_IN3_IDX(b, f, y) INPUT3_GET_INDEX(b, f, y, 0) + #define GET_IN4_IDX(b, f, y) INPUT4_GET_INDEX(b, f, y, 0) + #define GET_IN5_IDX(b, f) INPUT5_GET_INDEX(b, f, 0, 0) + #else + #define GET_IN1_IDX(b, f, y) INPUT1_GET_INDEX(b, 0, y, 0) + #define GET_IN2_IDX(b, f, y) INPUT2_GET_INDEX(b, 0, y, 0) + #define GET_IN3_IDX(b, f, y) INPUT3_GET_INDEX(0, f, y, 0) + #define GET_IN4_IDX(b, f, y) INPUT4_GET_INDEX(0, f, y, 0) + #define GET_IN5_IDX(b, f) INPUT5_GET_INDEX(0, f, 0, 0) + #endif +#else +#define GET_IN0_IDX(b, f, y) INPUT0_GET_INDEX(b, y, 0, 0) +#define GET_IN1_IDX(b, f, y) INPUT1_GET_INDEX(b, y, 0, 0) +#define GET_IN2_IDX(b, f, y) INPUT2_GET_INDEX(b, y, 0, 0) +#define GET_IN3_IDX(b, f, y) INPUT3_GET_INDEX(f, y, 0, 0) +#define GET_IN4_IDX(b, f, y) INPUT4_GET_INDEX(f, y, 0, 0) +#define GET_IN5_IDX(b, f) INPUT5_GET_INDEX(f, 0, 0, 0) +#endif + +KERNEL(lstm_cell_and_seq_bfyx)( + const __global INPUT0_TYPE* x, + const __global INPUT1_TYPE* initial_hidden_state, + const __global INPUT2_TYPE* initial_cell_state, + const __global INPUT3_TYPE* W, + const __global INPUT4_TYPE* R, + const __global INPUT5_TYPE* B, +#ifdef SEQUENCE + const __global INPUT6_TYPE* sequence_lengths, + __global OUTPUT_TYPE* hidden_history, + __global OUTPUT1_TYPE* hidden_state, + __global OUTPUT2_TYPE* cell_state +#else + __global OUTPUT_TYPE* hidden_state, + __global OUTPUT1_TYPE* cell_state +#endif +) +{ + const uint b = get_global_id(1); + const uint local_idx = get_local_id(0); + const uint weight_offsets[4] = {GEMM_OFFSET_F, GEMM_OFFSET_I, GEMM_OFFSET_Z, GEMM_OFFSET_O}; + #ifdef SEQUENCE + const uint real_seq_length = sequence_lengths[INPUT6_GET_INDEX(b, 0, 0, 0)]; + #else + const uint real_seq_length = 1; + #endif + #if DIRECTION == 2 + unroll_for(uint dir=0;dir0){ + barrier(CLK_LOCAL_MEM_FENCE); + } + #endif + unroll_for(uint l=0;l= HIDDEN_SIZE) { + continue; + } + ACCUMULATOR_TYPE gate_output[GATE_NUM]; + unroll_for(uint k=0;k0){ + barrier(CLK_LOCAL_MEM_FENCE); + } + #endif + unroll_for(uint l=0;l= HIDDEN_SIZE) { + continue; + } + ACCUMULATOR_TYPE gate_output[GATE_NUM]; + unroll_for(uint k=0;k

(tag, blob_path.blob, params.device); } - if (std::holds_alternative>>(params.reshape)) { - network->cfgReshape(std::get>>(params.reshape)); - } else { - network->cfgReshape(std::get>(params.reshape)); - } network->cfgPluginConfig(params.config); network->cfgNumRequests(params.nireq); diff --git a/src/plugins/template/tests/functional/op_reference/irdft.cpp b/src/plugins/template/tests/functional/op_reference/irdft.cpp index 36826ade32fb59..4bcf9fbbe27c18 100644 --- a/src/plugins/template/tests/functional/op_reference/irdft.cpp +++ b/src/plugins/template/tests/functional/op_reference/irdft.cpp @@ -29,8 +29,8 @@ struct IRDFTParams { m_expected_shape = expected_shape; m_input_type = input_type; m_expected_type = expected_type; - m_input_value = CreateTensor(input_type, input_value); - m_expected_value = CreateTensor(expected_type, expected_value); + m_input_value = CreateTensor(m_input_shape, input_type, input_value); + m_expected_value = CreateTensor(m_expected_shape, expected_type, expected_value); m_axes = axes; m_signal = signal; } @@ -48,8 +48,7 @@ struct IRDFTParams { class ReferenceIRDFTLayerTest : public testing::TestWithParam, public CommonReferenceTest { public: void SetUp() override { - legacy_compare = true; - auto params = GetParam(); + const auto& params = GetParam(); if (params.m_signal != NULL) { function = CreateFunctionWithSignal(params); } else { @@ -58,10 +57,12 @@ class ReferenceIRDFTLayerTest : public testing::TestWithParam, publ inputData = {params.m_input_value}; refOutData = {params.m_expected_value}; + + abs_threshold = 1e-6f; } static std::string getTestCaseName(const testing::TestParamInfo& obj) { - const auto param = obj.param; + const auto& param = obj.param; std::ostringstream result; result << "input_shape1=" << param.m_input_shape << "; "; @@ -74,14 +75,14 @@ class ReferenceIRDFTLayerTest : public testing::TestWithParam, publ } private: - static std::shared_ptr CreateFunction(IRDFTParams& p) { + static std::shared_ptr CreateFunction(const IRDFTParams& p) { auto in = std::make_shared(p.m_input_type, p.m_input_shape); auto irdft = std::make_shared(in, p.m_axes); return std::make_shared(irdft, ParameterVector{in}); } - static std::shared_ptr CreateFunctionWithSignal(IRDFTParams& p) { + static std::shared_ptr CreateFunctionWithSignal(const IRDFTParams& p) { auto in = std::make_shared(p.m_input_type, p.m_input_shape); auto irdft = std::make_shared(in, p.m_axes, p.m_signal); diff --git a/src/plugins/template/tests/functional/op_reference/reduce_l1.cpp b/src/plugins/template/tests/functional/op_reference/reduce_l1.cpp index 6e0c2fe2aa24e0..e47295f247b35f 100644 --- a/src/plugins/template/tests/functional/op_reference/reduce_l1.cpp +++ b/src/plugins/template/tests/functional/op_reference/reduce_l1.cpp @@ -25,6 +25,28 @@ std::vector generateReductionParams(const bool keep_dims) { reference_tests::Tensor(reduce(Shape{3, 2, 2}, AxisSet{2}, keep_dims), element::Type(IN_ET), std::vector{3, 7, 11, 15, 19, 23}))}; + auto out_shape_from_empty = Shape{2, 1, 1}; + if (keep_dims == false) { + out_shape_from_empty = Shape{2}; + } + params.push_back( + ReductionParams(ReductionType::L1, + keep_dims, + std::vector{1, 2}, + reference_tests::Tensor({2, 0, 4}, element::Type(IN_ET), std::vector{}), + reference_tests::Tensor(out_shape_from_empty, element::Type(IN_ET), std::vector{0, 0}))); + + out_shape_from_empty = Shape{2, 0, 1}; + if (keep_dims == false) { + out_shape_from_empty = Shape{2, 0}; + } + params.push_back( + ReductionParams(ReductionType::L1, + keep_dims, + std::vector{2}, + reference_tests::Tensor({2, 0, 4}, element::Type(IN_ET), std::vector{}), + reference_tests::Tensor(out_shape_from_empty, element::Type(IN_ET), std::vector{0, 0}))); + return params; } diff --git a/src/plugins/template/tests/functional/op_reference/reduce_l2.cpp b/src/plugins/template/tests/functional/op_reference/reduce_l2.cpp index 565f89d58f7238..b5820f6970ae5b 100644 --- a/src/plugins/template/tests/functional/op_reference/reduce_l2.cpp +++ b/src/plugins/template/tests/functional/op_reference/reduce_l2.cpp @@ -29,6 +29,27 @@ std::vector generateReductionParams(const bool keep_dims) { reference_tests::Tensor(reduce(Shape{3, 2, 2}, AxisSet{2}, keep_dims), element::Type(IN_ET), std::vector{2.23606798, 5.0, 7.81024968, 10.63014581, 13.45362405, 16.2788206}))}; + auto out_shape_from_empty = Shape{2, 1, 1}; + if (keep_dims == false) { + out_shape_from_empty = Shape{2}; + } + params.push_back( + ReductionParams(ReductionType::L2, + keep_dims, + std::vector{1, 2}, + reference_tests::Tensor({2, 0, 4}, element::Type(IN_ET), std::vector{}), + reference_tests::Tensor(out_shape_from_empty, element::Type(IN_ET), std::vector{0, 0}))); + + out_shape_from_empty = Shape{2, 0, 1}; + if (keep_dims == false) { + out_shape_from_empty = Shape{2, 0}; + } + params.push_back( + ReductionParams(ReductionType::L2, + keep_dims, + std::vector{2}, + reference_tests::Tensor({2, 0, 4}, element::Type(IN_ET), std::vector{}), + reference_tests::Tensor(out_shape_from_empty, element::Type(IN_ET), std::vector{}))); return params; } @@ -45,6 +66,7 @@ std::vector generateReductionParams(const bool keep_dims) { reference_tests::Tensor(reduce(Shape{3, 2, 2}, AxisSet{2}, keep_dims), element::Type(IN_ET), std::vector{2, 5, 8, 11, 13, 16}))}; + return params; } diff --git a/src/plugins/template/tests/functional/op_reference/reduce_max.cpp b/src/plugins/template/tests/functional/op_reference/reduce_max.cpp index 0674595de4ec43..7ab89fc16d1900 100644 --- a/src/plugins/template/tests/functional/op_reference/reduce_max.cpp +++ b/src/plugins/template/tests/functional/op_reference/reduce_max.cpp @@ -79,6 +79,29 @@ std::vector generateReductionParams(const bool keep_dims) { reference_tests::Tensor(reduce(Shape{3, 3, 3}, AxisSet{0, 1, 2}, keep_dims), element::Type(IN_ET), std::vector{27}))}; + auto out_shape_from_empty = Shape{2, 1, 1}; + if (keep_dims == false) { + out_shape_from_empty = Shape{2}; + } + const auto default_val = std::numeric_limits::lowest(); + params.push_back(ReductionParams( + ReductionType::Max, + keep_dims, + std::vector{1, 2}, + reference_tests::Tensor({2, 0, 4}, element::Type(IN_ET), std::vector{}), + reference_tests::Tensor(out_shape_from_empty, element::Type(IN_ET), std::vector{default_val, default_val}))); + + out_shape_from_empty = Shape{2, 0, 1}; + if (keep_dims == false) { + out_shape_from_empty = Shape{2, 0}; + } + params.push_back( + ReductionParams(ReductionType::Max, + keep_dims, + std::vector{2}, + reference_tests::Tensor({2, 0, 4}, element::Type(IN_ET), std::vector{}), + reference_tests::Tensor(out_shape_from_empty, element::Type(IN_ET), std::vector{}))); + return params; } diff --git a/src/plugins/template/tests/functional/op_reference/reduce_mean.cpp b/src/plugins/template/tests/functional/op_reference/reduce_mean.cpp index def9d837b46df6..07159de9704a30 100644 --- a/src/plugins/template/tests/functional/op_reference/reduce_mean.cpp +++ b/src/plugins/template/tests/functional/op_reference/reduce_mean.cpp @@ -41,6 +41,28 @@ std::vector generateReductionParams(const bool keep_dims) { reference_tests::Tensor(reduce(Shape{3, 2}, AxisSet{1}, keep_dims), element::Type(IN_ET), std::vector{1.5, 3.5, 5.5}))}; + auto out_shape_from_empty = Shape{2, 1, 1}; + if (keep_dims == false) { + out_shape_from_empty = Shape{2}; + } + params.push_back( + ReductionParams(ReductionType::Mean, + keep_dims, + std::vector{1, 2}, + reference_tests::Tensor({2, 0, 4}, element::Type(IN_ET), std::vector{}), + reference_tests::Tensor(out_shape_from_empty, element::Type(IN_ET), std::vector{0, 0}))); + + out_shape_from_empty = Shape{2, 0, 1}; + if (keep_dims == false) { + out_shape_from_empty = Shape{2, 0}; + } + params.push_back( + ReductionParams(ReductionType::Mean, + keep_dims, + std::vector{2}, + reference_tests::Tensor({2, 0, 4}, element::Type(IN_ET), std::vector{}), + reference_tests::Tensor(out_shape_from_empty, element::Type(IN_ET), std::vector{}))); + return params; } diff --git a/src/plugins/template/tests/functional/op_reference/reduce_min.cpp b/src/plugins/template/tests/functional/op_reference/reduce_min.cpp index abc9dca157684b..f982af07ab12a5 100644 --- a/src/plugins/template/tests/functional/op_reference/reduce_min.cpp +++ b/src/plugins/template/tests/functional/op_reference/reduce_min.cpp @@ -79,6 +79,29 @@ std::vector generateReductionParams(const bool keep_dims) { reference_tests::Tensor(reduce(Shape{3, 3, 3}, AxisSet{0, 1, 2}, keep_dims), element::Type(IN_ET), std::vector{1}))}; + auto out_shape_from_empty = Shape{2, 1, 1}; + if (keep_dims == false) { + out_shape_from_empty = Shape{2}; + } + constexpr auto max_value = + std::numeric_limits::has_infinity ? std::numeric_limits::infinity() : std::numeric_limits::max(); + params.push_back(ReductionParams( + ReductionType::Min, + keep_dims, + std::vector{1, 2}, + reference_tests::Tensor({2, 0, 4}, element::Type(IN_ET), std::vector{}), + reference_tests::Tensor(out_shape_from_empty, element::Type(IN_ET), std::vector{max_value, max_value}))); + + out_shape_from_empty = Shape{2, 0, 1}; + if (keep_dims == false) { + out_shape_from_empty = Shape{2, 0}; + } + params.push_back( + ReductionParams(ReductionType::Min, + keep_dims, + std::vector{2}, + reference_tests::Tensor({2, 0, 4}, element::Type(IN_ET), std::vector{}), + reference_tests::Tensor(out_shape_from_empty, element::Type(IN_ET), std::vector{}))); return params; } diff --git a/src/plugins/template/tests/functional/op_reference/reduce_prod.cpp b/src/plugins/template/tests/functional/op_reference/reduce_prod.cpp index d030633932fd73..54e39dad68826f 100644 --- a/src/plugins/template/tests/functional/op_reference/reduce_prod.cpp +++ b/src/plugins/template/tests/functional/op_reference/reduce_prod.cpp @@ -74,6 +74,28 @@ std::vector generateReductionParams(const bool keep_dims) { 19 * 20 * 21, 22 * 23 * 24, 25 * 26 * 27}))}; + auto out_shape_from_empty = Shape{2, 1, 1}; + if (keep_dims == false) { + out_shape_from_empty = Shape{2}; + } + const T default_val = T{1}; + params.push_back(ReductionParams( + ReductionType::Prod, + keep_dims, + std::vector{1, 2}, + reference_tests::Tensor({2, 0, 4}, element::Type(IN_ET), std::vector{}), + reference_tests::Tensor(out_shape_from_empty, element::Type(IN_ET), std::vector{default_val, default_val}))); + + out_shape_from_empty = Shape{2, 0, 1}; + if (keep_dims == false) { + out_shape_from_empty = Shape{2, 0}; + } + params.push_back( + ReductionParams(ReductionType::Prod, + keep_dims, + std::vector{2}, + reference_tests::Tensor({2, 0, 4}, element::Type(IN_ET), std::vector{}), + reference_tests::Tensor(out_shape_from_empty, element::Type(IN_ET), std::vector{}))); return params; } diff --git a/src/plugins/template/tests/functional/op_reference/reduce_sum.cpp b/src/plugins/template/tests/functional/op_reference/reduce_sum.cpp index ab77acc3cf696a..dd8dcd38635c79 100644 --- a/src/plugins/template/tests/functional/op_reference/reduce_sum.cpp +++ b/src/plugins/template/tests/functional/op_reference/reduce_sum.cpp @@ -121,6 +121,28 @@ std::vector generateReductionParams(const bool keep_dims) { reference_tests::Tensor(reduce(Shape{3, 3, 3, 3, 3}, AxisSet{0, 1, 2, 3, 4}, keep_dims), element::Type(IN_ET), std::vector{243}))}; + auto out_shape_from_empty = Shape{2, 1, 1}; + if (keep_dims == false) { + out_shape_from_empty = Shape{2}; + } + params.push_back( + ReductionParams(ReductionType::Sum, + keep_dims, + std::vector{1, 2}, + reference_tests::Tensor({2, 0, 4}, element::Type(IN_ET), std::vector{}), + reference_tests::Tensor(out_shape_from_empty, element::Type(IN_ET), std::vector{0, 0}))); + + out_shape_from_empty = Shape{2, 0, 1}; + if (keep_dims == false) { + out_shape_from_empty = Shape{2, 0}; + } + params.push_back( + ReductionParams(ReductionType::Sum, + keep_dims, + std::vector{2}, + reference_tests::Tensor({2, 0, 4}, element::Type(IN_ET), std::vector{}), + reference_tests::Tensor(out_shape_from_empty, element::Type(IN_ET), std::vector{}))); + return params; } diff --git a/src/tests/functional/plugin/shared/include/subgraph_tests/rotary_pos_emb.hpp b/src/tests/functional/plugin/shared/include/subgraph_tests/rotary_pos_emb.hpp index 7100ddca1083e3..c3f0b8ef0b6015 100644 --- a/src/tests/functional/plugin/shared/include/subgraph_tests/rotary_pos_emb.hpp +++ b/src/tests/functional/plugin/shared/include/subgraph_tests/rotary_pos_emb.hpp @@ -24,6 +24,13 @@ inline void CheckNumberOfNodesWithType(std::shared_ptr function ASSERT_EQ(num_ops, expectedCount); } +TEST_P(RoPETestFlux, CompareWithRefs) { + SKIP_IF_CURRENT_TEST_IS_DISABLED(); + run(); + auto function = compiledModel.get_runtime_model(); + CheckNumberOfNodesWithType(function, {"RoPE"}, 1); +}; + TEST_P(RoPETestLlama2StridedSlice, CompareWithRefs) { SKIP_IF_CURRENT_TEST_IS_DISABLED(); run(); diff --git a/src/tests/functional/plugin/shared/src/behavior/ov_infer_request/infer_request_dynamic.cpp b/src/tests/functional/plugin/shared/src/behavior/ov_infer_request/infer_request_dynamic.cpp index 1de6b4f610bbb8..f77a66063d6c98 100644 --- a/src/tests/functional/plugin/shared/src/behavior/ov_infer_request/infer_request_dynamic.cpp +++ b/src/tests/functional/plugin/shared/src/behavior/ov_infer_request/infer_request_dynamic.cpp @@ -281,12 +281,12 @@ TEST_P(OVInferRequestDynamicTests, InferDynamicNetworkWithGetTensor) { OV_ASSERT_NO_THROW(tensor = req.get_tensor(function->inputs().back().get_any_name())); OV_ASSERT_NO_THROW(tensor.set_shape({1, 4, 20, 20})); ASSERT_EQ(tensor.get_shape(), refShape); + std::memset(tensor.data(), 10.0f, tensor.get_byte_size()); + req.set_input_tensor(tensor); OV_ASSERT_NO_THROW(otensor = req.get_tensor(outputname)); ASSERT_EQ(0, otensor.get_size()); // output tensor is not allocated ASSERT_EQ(function->output(0).get_element_type(), otensor.get_element_type()); // by it has type OV_ASSERT_NO_THROW(req.infer()); - OV_ASSERT_NO_THROW(req.start_async()); - OV_ASSERT_NO_THROW(req.wait()); EXPECT_NE(0, otensor.get_size()); // output tensor is allocated after infer OV_ASSERT_NO_THROW(otensor = req.get_tensor(outputname)); ASSERT_EQ(otensor.get_shape(), refOutShape); @@ -432,9 +432,9 @@ TEST_P(OVInferRequestDynamicTests, InferDynamicNetworkWithGetTensor2times) { OV_ASSERT_NO_THROW(tensor = req.get_tensor(function->inputs().back().get_any_name())); OV_ASSERT_NO_THROW(tensor.set_shape(refShape)); ASSERT_EQ(tensor.get_shape(), refShape); + std::memset(tensor.data(), 10.0f, tensor.get_byte_size()); + req.set_input_tensor(tensor); OV_ASSERT_NO_THROW(req.infer()); - OV_ASSERT_NO_THROW(req.start_async()); - req.wait(); const std::string outputName = function->outputs().back().get_any_name(); OV_ASSERT_NO_THROW(tensor = req.get_tensor(outputName)); ASSERT_EQ(tensor.get_shape(), refOutShape); @@ -443,9 +443,9 @@ TEST_P(OVInferRequestDynamicTests, InferDynamicNetworkWithGetTensor2times) { OV_ASSERT_NO_THROW(tensor = req.get_tensor(function->inputs().back().get_any_name())); OV_ASSERT_NO_THROW(tensor.set_shape(refShape2)); ASSERT_EQ(tensor.get_shape(), refShape2); + std::memset(tensor.data(), 10.0f, tensor.get_byte_size()); + req.set_input_tensor(tensor); OV_ASSERT_NO_THROW(req.infer()); - OV_ASSERT_NO_THROW(req.start_async()); - req.wait(); OV_ASSERT_NO_THROW(tensor = req.get_tensor(outputName)); ASSERT_EQ(tensor.get_shape(), refOutShape2); ASSERT_TRUE(checkOutput(req.get_tensor("input_tensor"), req.get_tensor(outputName))); @@ -483,6 +483,7 @@ TEST_P(OVInferRequestDynamicTests, InferDynamicNetworkWithSetTensor) { // Create InferRequest ov::InferRequest req; ov::Tensor tensor(ov::element::f32, refShape); + std::memset(tensor.data(), 10.0f, tensor.get_byte_size()); OV_ASSERT_NO_THROW(req = execNet.create_infer_request()); OV_ASSERT_NO_THROW(req.set_tensor(function->inputs().back().get_any_name(), tensor)); ASSERT_EQ(tensor.get_shape(), refShape); @@ -507,6 +508,7 @@ TEST_P(OVInferRequestDynamicTests, InferFullyDynamicNetworkWithSetTensor) { // Create InferRequest ov::InferRequest req; ov::Tensor tensor(ov::element::f32, refShape), otensor; + std::memset(tensor.data(), 10.0f, tensor.get_byte_size()); const std::string outputName = function->outputs().back().get_any_name(); OV_ASSERT_NO_THROW(req = execNet.create_infer_request()); OV_ASSERT_NO_THROW(req.set_tensor(function->inputs().back().get_any_name(), tensor)); @@ -540,23 +542,20 @@ TEST_P(OVInferRequestDynamicTests, InferDynamicNetworkWithSetTensor2times) { // Create InferRequest ov::InferRequest req; ov::Tensor tensor(ov::element::f32, refShape); - + std::memset(tensor.data(), 10.0f, tensor.get_byte_size()); OV_ASSERT_NO_THROW(req = execNet.create_infer_request()); OV_ASSERT_NO_THROW(req.set_tensor(function->inputs().back().get_any_name(), tensor)); ASSERT_EQ(tensor.get_shape(), refShape); OV_ASSERT_NO_THROW(req.infer()); - OV_ASSERT_NO_THROW(req.start_async()); - OV_ASSERT_NO_THROW(req.wait()); OV_ASSERT_NO_THROW(tensor = req.get_tensor(outputName)); ASSERT_EQ(tensor.get_shape(), refOutShape); ASSERT_TRUE(checkOutput(req.get_tensor("input_tensor"), req.get_tensor(outputName))); tensor = ov::Tensor(ov::element::f32, refShape2); + std::memset(tensor.data(), 10.0f, tensor.get_byte_size()); OV_ASSERT_NO_THROW(req.set_tensor(function->inputs().back().get_any_name(), tensor)); ASSERT_EQ(tensor.get_shape(), refShape2); OV_ASSERT_NO_THROW(req.infer()); - OV_ASSERT_NO_THROW(req.start_async()); - OV_ASSERT_NO_THROW(req.wait()); OV_ASSERT_NO_THROW(tensor = req.get_tensor(outputName)); ASSERT_EQ(tensor.get_shape(), refOutShape2); ASSERT_TRUE(checkOutput(req.get_tensor("input_tensor"), req.get_tensor(outputName))); diff --git a/src/tests/functional/plugin/shared/src/behavior/ov_plugin/caching_tests.cpp b/src/tests/functional/plugin/shared/src/behavior/ov_plugin/caching_tests.cpp index 73cefa6cc4d8d1..39ed6d9b6bacbd 100644 --- a/src/tests/functional/plugin/shared/src/behavior/ov_plugin/caching_tests.cpp +++ b/src/tests/functional/plugin/shared/src/behavior/ov_plugin/caching_tests.cpp @@ -380,7 +380,10 @@ TEST_P(CompileModelLoadFromFileTestBase, CanCreateCacheDirAndDumpBinariesUnicode // Check that directory with cached model exists after loading network ASSERT_TRUE(ov::util::directory_exists(cache_path_w)) << "Directory with cached kernels doesn't exist"; // Check that folder contains cache files and remove them - ASSERT_GT(ov::test::utils::removeFilesWithExt(cache_path_w, ov::util::string_to_wstring("blob")), 0); + int removed_files_num = 0; + removed_files_num += ov::test::utils::removeFilesWithExt(cache_path_w, ov::util::string_to_wstring("blob")); + removed_files_num += ov::test::utils::removeFilesWithExt(cache_path_w, ov::util::string_to_wstring("cl_cache")); + ASSERT_GT(removed_files_num, 0); ov::test::utils::removeFile(model_xml_path_w); ov::test::utils::removeFile(model_bin_path_w); // Remove directory and check that it doesn't exist anymore @@ -786,10 +789,12 @@ TEST_P(CompiledKernelsCacheTest, CanCreateCacheDirAndDumpBinaries) { // Check that directory with cached kernels exists after loading network ASSERT_TRUE(ov::util::directory_exists(cache_path)) << "Directory with cached kernels doesn't exist"; // Check that folder contains cache files and remove them + int number_of_deleted_files = 0; for (auto& ext : m_extList) { // Check that folder contains cache files and remove them - ASSERT_GT(ov::test::utils::removeFilesWithExt(cache_path, ext), 0); + number_of_deleted_files += ov::test::utils::removeFilesWithExt(cache_path, ext); } + ASSERT_GT(number_of_deleted_files, 0); // Remove directory and check that it doesn't exist anymore ASSERT_EQ(ov::test::utils::removeDir(cache_path), 0); ASSERT_FALSE(ov::util::directory_exists(cache_path)); @@ -824,12 +829,14 @@ TEST_P(CompiledKernelsCacheTest, TwoNetworksWithSameModelCreatesSameCache) { auto execNet2 = core->compile_model(function, targetDevice, configuration); execNet2 = {}; size_t n_cache_files_compare = 0; + int number_of_deleted_files = 0; // Check that two loaded networks with same function creates same caches for (auto& ext : m_extList) { // Check that folder contains cache files and remove them n_cache_files_compare += ov::test::utils::listFilesWithExt(cache_path, ext).size(); - ASSERT_TRUE(ov::test::utils::removeFilesWithExt(cache_path, ext)); + number_of_deleted_files += ov::test::utils::removeFilesWithExt(cache_path, ext); } + ASSERT_GT(number_of_deleted_files, 0); ASSERT_EQ(n_cache_files_compare, n_cache_files); // Remove directory and check that it doesn't exist anymore @@ -865,10 +872,12 @@ TEST_P(CompiledKernelsCacheTest, CanCreateCacheDirAndDumpBinariesUnicodePath) { // Check that directory with cached kernels exists after loading network ASSERT_TRUE(ov::util::directory_exists(cache_path_w)) << "Directory with cached kernels doesn't exist"; // Check that folder contains cache files and remove them + int count_of_removed_files = 0; for (auto& ext : m_extList) { // Check that folder contains cache files and remove them - ASSERT_GT(ov::test::utils::removeFilesWithExt(cache_path_w, ov::test::utils::stringToWString(ext)), 0); + count_of_removed_files += ov::test::utils::removeFilesWithExt(cache_path_w, ov::test::utils::stringToWString(ext)); } + ASSERT_GT(count_of_removed_files, 0); // Remove directory and check that it doesn't exist anymore ASSERT_EQ(ov::test::utils::removeDir(cache_path_w), 0); ASSERT_FALSE(ov::util::directory_exists(cache_path_w)); diff --git a/src/tests/functional/shared_test_classes/include/shared_test_classes/subgraph/rotary_pos_emb.hpp b/src/tests/functional/shared_test_classes/include/shared_test_classes/subgraph/rotary_pos_emb.hpp index e1182bd3b16e13..39cdb871710e64 100644 --- a/src/tests/functional/shared_test_classes/include/shared_test_classes/subgraph/rotary_pos_emb.hpp +++ b/src/tests/functional/shared_test_classes/include/shared_test_classes/subgraph/rotary_pos_emb.hpp @@ -9,6 +9,20 @@ namespace ov { namespace test { +class RoPETestFlux : public SubgraphBaseTest, public testing::WithParamInterface { +private: + std::shared_ptr build_rope_flux(int batch, + int seq_length, + int num_head, + int ndims); +protected: + void generate_inputs(const std::vector& targetInputStaticShapes) override; + void SetUp() override; + +public: + static std::string getTestCaseName(const testing::TestParamInfo& obj); +}; + class RoPETestLlama2StridedSlice : public SubgraphBaseTest, public testing::WithParamInterface { private: std::shared_ptr buildROPE_Llama2(int batch, diff --git a/src/tests/functional/shared_test_classes/src/single_op/search_sorted.cpp b/src/tests/functional/shared_test_classes/src/single_op/search_sorted.cpp index a92d87d51f9a10..c7c10ad8767ff6 100644 --- a/src/tests/functional/shared_test_classes/src/single_op/search_sorted.cpp +++ b/src/tests/functional/shared_test_classes/src/single_op/search_sorted.cpp @@ -88,11 +88,30 @@ void SearchSortedLayerTest::SetUp() { const std::vector SearchSortedLayerTest::GenerateParams() { const std::vector params = { + SearchSortedSpecificParams{InputShape{PartialShape::dynamic(3), {{1, 18, 104}}}, + InputShape{PartialShape::dynamic(3), {{1, 18, 104}}}, + true}, + SearchSortedSpecificParams{InputShape{PartialShape::dynamic(4), {{1, 2, 3, 100}}}, + InputShape{PartialShape::dynamic(4), {{1, 2, 3, 10}}}, + true}, + SearchSortedSpecificParams{InputShape{PartialShape::dynamic(5), {{2, 1, 2, 3, 10}}}, + InputShape{PartialShape::dynamic(5), {{2, 1, 2, 3, 20}}}, + false}, + SearchSortedSpecificParams{InputShape{PartialShape::dynamic(1), {{1}}}, + InputShape{PartialShape::dynamic(5), {{2, 1, 2, 3, 20}}}, + false}, + SearchSortedSpecificParams{InputShape{PartialShape::dynamic(1), {{50}}}, + InputShape{{1, -1, 10}, {{1, 18, 10}}}, + false}, SearchSortedSpecificParams{InputShape{{}, {{1, 18, 104}}}, InputShape{{}, {{1, 18, 104}}}, true}, SearchSortedSpecificParams{InputShape{{}, {{1, 2, 3, 100}}}, InputShape{{}, {{1, 2, 3, 10}}}, true}, SearchSortedSpecificParams{InputShape{{}, {{2, 1, 2, 3, 10}}}, InputShape{{}, {{2, 1, 2, 3, 20}}}, false}, SearchSortedSpecificParams{InputShape{{}, {{1}}}, InputShape{{}, {{2, 1, 2, 3, 20}}}, false}, SearchSortedSpecificParams{InputShape{{}, {{50}}}, InputShape{{1, -1, 10}, {{1, 18, 10}}}, false}, + SearchSortedSpecificParams{InputShape{{2, -1, 50}, {{2, 3, 50}}}, + InputShape{{-1, -1, 10}, {{2, 3, 10}}}, + false}, + SearchSortedSpecificParams{InputShape{{2, -1, 50}, {{2, 3, 50}}}, InputShape{{-1, 3, 10}, {{2, 3, 10}}}, false}, }; return params; diff --git a/src/tests/functional/shared_test_classes/src/subgraph/rotary_pos_emb.cpp b/src/tests/functional/shared_test_classes/src/subgraph/rotary_pos_emb.cpp index a1848903bb76a2..1a078d9b49ebb7 100644 --- a/src/tests/functional/shared_test_classes/src/subgraph/rotary_pos_emb.cpp +++ b/src/tests/functional/shared_test_classes/src/subgraph/rotary_pos_emb.cpp @@ -5,6 +5,7 @@ #include "shared_test_classes/subgraph/rotary_pos_emb.hpp" #include "common_test_utils/ov_tensor_utils.hpp" +#include "openvino/core/node_vector.hpp" #include "transformations/utils/gen_pattern.hpp" using namespace ov::gen_pattern; @@ -13,6 +14,85 @@ using namespace ov; namespace ov { namespace test { +std::shared_ptr RoPETestFlux::build_rope_flux(int batch, + int seq_length, + int num_head, + int ndims) { + auto x = std::make_shared(ov::element::f32, PartialShape{batch, num_head, seq_length, ndims}); + auto t_cos = std::make_shared(ov::element::f32, PartialShape{1, 1, seq_length, ndims}); + auto t_sin = std::make_shared(ov::element::f32, PartialShape{1, 1, seq_length, ndims}); + + auto x1_shape = makeConst(element::i64, ov::Shape({5}), {0, num_head, 0, -1, 2}); + auto x1 = std::make_shared(x, x1_shape, true); + + auto split_axis = makeConst(element::i64, ov::Shape(), {-1}); + auto split = std::make_shared(x1, split_axis, 2); + + auto minus_one = makeConst(element::f32, ov::Shape({}), {-1.0f}); + auto x1_1_neg = std::make_shared(split->output(1), minus_one); + + auto x2 = std::make_shared(OutputVector{x1_1_neg->output(0), split->output(0)}, -1); + + auto x3_shape = makeConst(element::i64, ov::Shape({4}), {0, num_head, 0, ndims}); + auto x3 = std::make_shared(x2, x3_shape, true); + + auto y1 = std::make_shared(x, t_cos); + auto y2 = std::make_shared(x3, t_sin); + auto y = std::make_shared(y1, y2); + + return std::make_shared(ov::NodeVector{y}, ov::ParameterVector{x, t_cos, t_sin}); +} + +void RoPETestFlux::generate_inputs(const std::vector& targetInputStaticShapes) { + const auto& funcInputs = function->inputs(); + + ov::test::utils::InputGenerateData in_data; + in_data.start_from = -1; + in_data.range = 2; + in_data.resolution = 32768; + + auto cos_data = in_data; + cos_data.seed = 10; + + auto sin_data = in_data; + sin_data.seed = 20; + + ov::Tensor t_input = utils::create_and_fill_tensor(funcInputs[0].get_element_type(), targetInputStaticShapes[0], in_data); + ov::Tensor t_cos = utils::create_and_fill_tensor(funcInputs[1].get_element_type(), targetInputStaticShapes[1], cos_data); + ov::Tensor t_sin = utils::create_and_fill_tensor(funcInputs[2].get_element_type(), targetInputStaticShapes[2], sin_data); + + inputs.clear(); + inputs.insert({funcInputs[0].get_node_shared_ptr(), t_input}); + inputs.insert({funcInputs[1].get_node_shared_ptr(), t_cos}); + inputs.insert({funcInputs[2].get_node_shared_ptr(), t_sin}); +} + +void RoPETestFlux::SetUp() { + targetDevice = this->GetParam(); + + const int batch = 1; + const int seq_length = 7; + const size_t max_position_embeddings = 2048; + const size_t ndims = 128; + const size_t num_head = 24; + + std::vector input_shapes = { + {{batch, num_head, seq_length, ndims}, {{batch, num_head, seq_length, ndims}}}, + {{1, 1, seq_length, ndims}, {{1, 1, seq_length, ndims}}}, + {{1, 1, seq_length, ndims}, {{1, 1, seq_length, ndims}}} + }; + init_input_shapes(input_shapes); + function = build_rope_flux(batch, -1, num_head, ndims); +} + +std::string RoPETestFlux::getTestCaseName(const testing::TestParamInfo& obj) { + std::string targetDevice = obj.param; + std::ostringstream result; + result << "targetDevice=" << targetDevice; + return result.str(); +} + + ov::OutputVector RoPETestLlama2StridedSlice::makeCosSinCache(int max_position_embeddings, int rotary_ndims) { std::vector lut_sin(max_position_embeddings * rotary_ndims, 0.0f); std::vector lut_cos(max_position_embeddings * rotary_ndims, 0.0f); diff --git a/src/tests/test_utils/unit_test_utils/tests_data/search_sorted_data.h b/src/tests/test_utils/unit_test_utils/tests_data/search_sorted_data.h index ee355c2daee15e..43e680aa080686 100644 --- a/src/tests/test_utils/unit_test_utils/tests_data/search_sorted_data.h +++ b/src/tests/test_utils/unit_test_utils/tests_data/search_sorted_data.h @@ -13,6 +13,22 @@ // NOTE: expected output were generated using pyTorch.searchsorted implementation. +TEST_DATA(LIST(5), + LIST(2, 3), + false, + LIST(3, 3, 3, 3, 3), + LIST(3, 6, 9, 3, 6, 9), + LIST(0, 5, 5, 0, 5, 5), + "1d_tensor_0"); + +TEST_DATA(LIST(5), + LIST(2, 3), + true, + LIST(3, 3, 3, 3, 3), + LIST(3, 6, 9, 3, 6, 9), + LIST(5, 5, 5, 5, 5, 5), + "1d_tensor_0_right_mode"); + TEST_DATA(LIST(5), LIST(2, 3), false, @@ -53,6 +69,22 @@ TEST_DATA(LIST(5), LIST(0, 3, 5, 1, 3, 5, 1, 0, 0, 5, 5, 5), "1d_tensor_3_right_mode"); +TEST_DATA(LIST(1), + LIST(2, 2, 3), + false, + LIST(2), + LIST(0, 6, 20, 2, 6, 9, 1, 0, 0, 9, 10, 20), + LIST(0, 1, 1, 0, 1, 1, 0, 0, 0, 1, 1, 1), + "1d_tensor_4"); + +TEST_DATA(LIST(1), + LIST(2, 2, 3), + true, + LIST(2), + LIST(0, 6, 20, 2, 6, 9, 1, 0, 0, 9, 10, 20), + LIST(0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1), + "1d_tensor_4_right_mode"); + TEST_DATA(LIST(2, 5), LIST(2, 3), false, @@ -72,15 +104,15 @@ TEST_DATA(LIST(2, 5), TEST_DATA(LIST(2, 2, 5), LIST(2, 2, 3), false, - LIST(1, 3, 5, 7, 9, 0, 2, 4, 6, 8, -20, 5, 10, 23, 41, 100, 125, 130, 132, 139), + LIST(1, 3, 5, 7, 9, 0, 2, 4, 6, 8, 0, 5, 10, 23, 41, 100, 125, 130, 132, 139), LIST(0, 6, 20, 1, 6, 9, 1, 0, 0, 9, 10, 20), - LIST(0, 3, 5, 1, 3, 5, 1, 1, 1, 0, 0, 0), + LIST(0, 3, 5, 1, 3, 5, 1, 0, 0, 0, 0, 0), "nd_tensor_2"); TEST_DATA(LIST(2, 2, 5), LIST(2, 2, 3), true, - LIST(1, 3, 5, 7, 9, 0, 2, 4, 6, 8, -20, 5, 10, 23, 41, 100, 125, 130, 132, 139), + LIST(1, 3, 5, 7, 9, 0, 2, 4, 6, 8, 0, 5, 10, 23, 41, 100, 125, 130, 132, 139), LIST(0, 6, 20, 1, 6, 9, 1, 0, 0, 9, 10, 20), LIST(0, 3, 5, 1, 4, 5, 1, 1, 1, 0, 0, 0), - "nd_tensor_2"); \ No newline at end of file + "nd_tensor_2_right_mode"); \ No newline at end of file diff --git a/tests/layer_tests/pytorch_tests/test_index_fill_.py b/tests/layer_tests/pytorch_tests/test_index_fill_.py new file mode 100644 index 00000000000000..878dda7ab3bd7e --- /dev/null +++ b/tests/layer_tests/pytorch_tests/test_index_fill_.py @@ -0,0 +1,83 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + + +import numpy as np +import pytest +import torch + +from pytorch_layer_test_class import PytorchLayerTest + + +class TestIndexFill(PytorchLayerTest): + def _prepare_input(self): + return (self.input_tensor,) + + def create_model(self, dim, index, values): + class aten_index_fill_(torch.nn.Module): + def __init__(self, dim, index, values): + super().__init__() + self.dim = dim + self.index = index + self.values = values + + def forward(self, input_tensor): + input_tensor.index_fill_(self.dim, self.index, self.values) + return input_tensor + + ref_net = None + + return aten_index_fill_(dim, index, values), ref_net, "aten::index_fill_" + + @pytest.mark.parametrize( + "input_data", + ( + { + "input_shape": [10], + "dim": 0, + "input_value": 5.6, + "index": [5, 6, 7] + }, + { + "input_shape": [3, 3], + "dim": 0, + "input_value": 10.1, + "index": [1, 0] + }, + { + "input_shape": [4, 3, 5], + "dim": 1, + "input_value": 1234.5, + "index": [2, 0] + }, + { + "input_shape": [5, 6, 7, 8], + "dim": -2, + "input_value": 0.1234, + "index": [6, 4, 2, 0] + }, + { + "input_shape": [5, 6, 7, 8], + "dim": -3, + "input_value": -4321234.5678765, + "index": [5, 4, 3, 1] + }, + { + "input_shape": [5, 6, 7, 8], + "dim": 3, + "input_value": -1234.54321, + "index": [6, 4, 7, 2, 1] + }, + ), + ) + @pytest.mark.nightly + @pytest.mark.precommit + def test_index_fill_single_index(self, ie_device, precision, ir_version, input_data): + self.input_tensor = np.random.randn(*input_data["input_shape"]).astype(np.float32) + values = torch.tensor(np.float32(input_data["input_value"])) + dim = input_data["dim"] + shape = self.input_tensor.shape + max_idx = shape[dim] + n_select = np.random.randint(1, max_idx + 1) + index = torch.from_numpy(np.random.choice(np.arange(0, max_idx), n_select, replace=False)).to(torch.long) + self._test(*self.create_model(dim, index, values), ie_device, precision, ir_version) \ No newline at end of file diff --git a/tests/requirements_onnx b/tests/requirements_onnx index 1dfc0077b5d075..9772bbf1ea0d52 100644 --- a/tests/requirements_onnx +++ b/tests/requirements_onnx @@ -1,3 +1,3 @@ -numpy>=1.16.6,<1.27 +numpy>=1.16.6,<2.2.0 onnx>=1.8.1,<=1.17.0 protobuf>=3.18.1,<4.0.0 diff --git a/tools/ovc/openvino/tools/ovc/__init__.py b/tools/ovc/openvino/tools/ovc/__init__.py index 5b750b58969d24..3785d45324713f 100644 --- a/tools/ovc/openvino/tools/ovc/__init__.py +++ b/tools/ovc/openvino/tools/ovc/__init__.py @@ -1,8 +1,9 @@ # Copyright (C) 2018-2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +import sys from openvino.tools.ovc.convert import convert_model -from openvino.tools.ovc.telemetry_utils import is_optimum, init_ovc_telemetry +from openvino.tools.ovc.telemetry_utils import is_optimum, init_ovc_telemetry, is_torch_compile import importlib.metadata as importlib_metadata @@ -11,10 +12,15 @@ except importlib_metadata.PackageNotFoundError: optimum_version = None -from openvino.runtime import get_version as get_rt_version # pylint: disable=no-name-in-module,import-error +from openvino import get_version as get_rt_version # pylint: disable=no-name-in-module,import-error telemetry = init_ovc_telemetry('OpenVINO') telemetry.send_event("ov", "import", "general_import") if is_optimum() and optimum_version is not None: telemetry = init_ovc_telemetry("Optimum Intel", optimum_version) telemetry.send_event("optimum", "import", "import_from_optimum,ov_version:{}".format(get_rt_version())) + +if is_torch_compile() and 'torch' in sys.modules: + torch_version = importlib_metadata.version("torch") + telemetry = init_ovc_telemetry("torch.compile", torch_version) + telemetry.send_event("torch.compile", "import", "Import from torch.compile(), ov_version: {}".format(get_rt_version())) diff --git a/tools/ovc/openvino/tools/ovc/telemetry_utils.py b/tools/ovc/openvino/tools/ovc/telemetry_utils.py index 412d9b9607541e..f68a92be5d2de5 100644 --- a/tools/ovc/openvino/tools/ovc/telemetry_utils.py +++ b/tools/ovc/openvino/tools/ovc/telemetry_utils.py @@ -24,6 +24,12 @@ def is_optimum(): return True return False +def is_torch_compile(): + import traceback + for line in traceback.format_stack(): + if os.path.join("torch", "_dynamo", "backends", "registry.py") in line: + return True + return False def init_ovc_telemetry(app_name='OVC', app_version=None): app_version = app_version if app_version is not None else get_rt_version()