Reserving CPU resource in CPU inference

src/inference/include/openvino/runtime/properties.hpp

@@ -479,6 +479,24 @@ static constexpr Property<std::set<ModelDistributionPolicy>> model_distribution_
  */
 static constexpr Property<bool> enable_cpu_pinning{"ENABLE_CPU_PINNING"};
 
+/**
+ * @brief This property allows CPU reservation during inference.
+ * @ingroup ov_runtime_cpp_prop_api
+ *
+ * Cpu Reservation means reserve cpus which will not be used by other plugin. Developer can use this property to
+ * enable or disable CPU reservation during inference on Windows and Linux. MacOS
+ * does not support CPU reservation, and this property is always disabled.
+ * This property defaults to false.
+ *
+ * The following code is example to use this property.
+ *
+ * @code
+ * ie.set_property(ov::hint::enable_cpu_reservation(true));
+ * ie.set_property(ov::hint::enable_cpu_reservation(false));
+ * @endcode
+ */
+static constexpr Property<bool> enable_cpu_reservation{"ENABLE_CPU_RESERVATION"};
+
 /**
  * @brief This property define if using hyper threading during inference.
  * @ingroup ov_runtime_cpp_prop_api

src/inference/src/dev/threading/istreams_executor.cpp

@@ -163,10 +163,6 @@ void IStreamsExecutor::Config::update_executor_config() {
         return;
     }
 
-    if (_cpu_reservation && !_cpu_pinning) {
-        _cpu_pinning = true;
-    }
-
     if (!_streams_info_table.empty()) {
         streams_info_available = true;
         std::vector<int> threads_proc_type(HYPER_THREADING_PROC + 1, 0);
@@ -265,7 +261,7 @@ void IStreamsExecutor::Config::update_executor_config() {
         }
     }
 
-    if (_cpu_pinning) {
+    if (_cpu_pinning || _cpu_reservation) {
         reserve_available_cpus(_streams_info_table, _stream_processor_ids, _cpu_reservation ? CPU_USED : NOT_USED);
     }
 

src/plugins/intel_cpu/src/compiled_model.cpp

@@ -247,6 +247,7 @@ ov::Any CompiledModel::get_property(const std::string& name) const {
             RO_property(ov::hint::execution_mode.name()),
             RO_property(ov::hint::num_requests.name()),
             RO_property(ov::hint::enable_cpu_pinning.name()),
+            RO_property(ov::hint::enable_cpu_reservation.name()),
             RO_property(ov::hint::scheduling_core_type.name()),
             RO_property(ov::hint::model_distribution_policy.name()),
             RO_property(ov::hint::enable_hyper_threading.name()),
@@ -307,6 +308,9 @@ ov::Any CompiledModel::get_property(const std::string& name) const {
     } else if (name == ov::hint::enable_cpu_pinning.name()) {
         const bool use_pin = config.enableCpuPinning;
         return decltype(ov::hint::enable_cpu_pinning)::value_type(use_pin);
+    } else if (name == ov::hint::enable_cpu_reservation.name()) {
+        const bool use_reserve = config.enableCpuReservation;
+        return decltype(ov::hint::enable_cpu_reservation)::value_type(use_reserve);
     } else if (name == ov::hint::scheduling_core_type) {
         const auto stream_mode = config.schedulingCoreType;
         return stream_mode;

src/plugins/intel_cpu/src/config.cpp

@@ -178,6 +178,16 @@ void Config::readProperties(const ov::AnyMap& prop, const ModelType modelType) {
                                ov::hint::enable_cpu_pinning.name(),
                                ". Expected only true/false.");
             }
+        } else if (key == ov::hint::enable_cpu_reservation.name()) {
+            try {
+                enableCpuReservation = val.as<bool>();
+            } catch (ov::Exception&) {
+                OPENVINO_THROW("Wrong value ",
+                               val.as<std::string>(),
+                               "for property key ",
+                               ov::hint::enable_cpu_reservation.name(),
+                               ". Expected only true/false.");
+            }
         } else if (key == ov::hint::scheduling_core_type.name()) {
             try {
                 schedulingCoreType = val.as<ov::hint::SchedulingCoreType>();

src/plugins/intel_cpu/src/config.h

@@ -75,6 +75,7 @@ struct Config {
     uint32_t hintNumRequests = 0;
     bool enableCpuPinning = true;
     bool changedCpuPinning = false;
+    bool enableCpuReservation = false;
     ov::hint::SchedulingCoreType schedulingCoreType = ov::hint::SchedulingCoreType::ANY_CORE;
     std::set<ov::hint::ModelDistributionPolicy> modelDistributionPolicy = {};
     int streamsRankLevel = 1;

src/plugins/intel_cpu/src/cpu_streams_calculation.cpp

@@ -691,7 +691,7 @@ std::vector<std::vector<int>> generate_stream_info(const int streams,
                                                            config.streams,
                                                            config.threadsPerStream,
                                                            ov::hint::SchedulingCoreType::ANY_CORE,
-                                                           false,
+                                                           config.enableCpuReservation,
                                                            cpu_pinning,
                                                            streams_info_table};
 

src/plugins/intel_cpu/src/plugin.cpp

@@ -346,6 +346,9 @@ ov::Any Plugin::get_property(const std::string& name, const ov::AnyMap& options)
     } else if (name == ov::hint::enable_cpu_pinning) {
         const bool pin_value = engConfig.enableCpuPinning;
         return decltype(ov::hint::enable_cpu_pinning)::value_type(pin_value);
+    } else if (name == ov::hint::enable_cpu_reservation) {
+        const bool reserve_value = engConfig.enableCpuReservation;
+        return decltype(ov::hint::enable_cpu_reservation)::value_type(reserve_value);
     } else if (name == ov::hint::scheduling_core_type) {
         const auto core_type = engConfig.schedulingCoreType;
         return core_type;
@@ -422,6 +425,7 @@ ov::Any Plugin::get_ro_property(const std::string& name, const ov::AnyMap& optio
             RW_property(ov::hint::execution_mode.name()),
             RW_property(ov::hint::num_requests.name()),
             RW_property(ov::hint::enable_cpu_pinning.name()),
+            RW_property(ov::hint::enable_cpu_reservation.name()),
             RW_property(ov::hint::scheduling_core_type.name()),
             RW_property(ov::hint::model_distribution_policy.name()),
             RW_property(ov::hint::enable_hyper_threading.name()),

src/plugins/intel_cpu/tests/functional/custom/behavior/ov_executable_network/properties.cpp

@@ -32,6 +32,7 @@ TEST_F(OVClassConfigTestCPU, smoke_CpuExecNetworkSupportedPropertiesAreAvailable
         RO_property(ov::hint::execution_mode.name()),
         RO_property(ov::hint::num_requests.name()),
         RO_property(ov::hint::enable_cpu_pinning.name()),
+        RO_property(ov::hint::enable_cpu_reservation.name()),
         RO_property(ov::hint::scheduling_core_type.name()),
         RO_property(ov::hint::model_distribution_policy.name()),
         RO_property(ov::hint::enable_hyper_threading.name()),

src/plugins/intel_cpu/tests/functional/custom/behavior/ov_plugin/properties.cpp

@@ -47,6 +47,7 @@ TEST_F(OVClassConfigTestCPU, smoke_PluginAllSupportedPropertiesAreAvailable) {
         RW_property(ov::hint::execution_mode.name()),
         RW_property(ov::hint::num_requests.name()),
         RW_property(ov::hint::enable_cpu_pinning.name()),
+        RW_property(ov::hint::enable_cpu_reservation.name()),
         RW_property(ov::hint::scheduling_core_type.name()),
         RW_property(ov::hint::model_distribution_policy.name()),
         RW_property(ov::hint::enable_hyper_threading.name()),

src/plugins/intel_gpu/src/graph/program.cpp

@@ -113,11 +113,13 @@ static ov::threading::IStreamsExecutor::Config make_task_executor_config(const E
         default: OPENVINO_ASSERT(false, "[GPU] Can't create task executor: invalid host task priority value: ", priority);
     }
     bool enable_cpu_pinning = config.get_property(ov::hint::enable_cpu_pinning);
+    bool enable_cpu_reservation = config.get_property(ov::hint::enable_cpu_reservation);
 
     ov::threading::IStreamsExecutor::Config task_executor_config(tags,
                                                                  streams,
                                                                  1,
                                                                  core_type,
+                                                                 enable_cpu_reservation,
                                                                  enable_cpu_pinning);
 
     return task_executor_config;

src/plugins/intel_gpu/src/plugin/compiled_model.cpp

@@ -25,11 +25,13 @@ std::shared_ptr<ov::threading::ITaskExecutor> create_task_executor(const std::sh
         // the CPU behavior
         return plugin->get_executor_manager()->get_executor("GPU");
     } else if (config.get_property(ov::hint::enable_cpu_pinning)) {
+        bool enable_cpu_reservation = config.get_property(ov::hint::enable_cpu_reservation);
         return std::make_shared<ov::threading::CPUStreamsExecutor>(
             ov::threading::IStreamsExecutor::Config{"Intel GPU plugin executor",
                                                     config.get_property(ov::num_streams),
                                                     1,
                                                     ov::hint::SchedulingCoreType::PCORE_ONLY,
+                                                    enable_cpu_reservation,
                                                     true});
     } else {
         return std::make_shared<ov::threading::CPUStreamsExecutor>(
@@ -242,6 +244,7 @@ ov::Any CompiledModel::get_property(const std::string& name) const {
             // Configs
             ov::PropertyName{ov::enable_profiling.name(), PropertyMutability::RO},
             ov::PropertyName{ov::hint::enable_cpu_pinning.name(), PropertyMutability::RO},
+            ov::PropertyName{ov::hint::enable_cpu_reservation.name(), PropertyMutability::RO},
             ov::PropertyName{ov::hint::model_priority.name(), PropertyMutability::RO},
             ov::PropertyName{ov::intel_gpu::hint::host_task_priority.name(), PropertyMutability::RO},
             ov::PropertyName{ov::intel_gpu::hint::queue_priority.name(), PropertyMutability::RO},

src/plugins/intel_gpu/src/plugin/plugin.cpp

@@ -564,6 +564,7 @@ std::vector<ov::PropertyName> Plugin::get_supported_properties() const {
         ov::PropertyName{ov::hint::num_requests.name(), PropertyMutability::RW},
         ov::PropertyName{ov::hint::inference_precision.name(), PropertyMutability::RW},
         ov::PropertyName{ov::hint::enable_cpu_pinning.name(), PropertyMutability::RW},
+        ov::PropertyName{ov::hint::enable_cpu_reservation.name(), PropertyMutability::RW},
         ov::PropertyName{ov::device::id.name(), PropertyMutability::RW},
         ov::PropertyName{ov::hint::dynamic_quantization_group_size.name(), PropertyMutability::RW}
     };

src/plugins/intel_gpu/src/runtime/execution_config.cpp

@@ -46,6 +46,7 @@ void ExecutionConfig::set_default() {
         std::make_tuple(ov::hint::execution_mode, ov::hint::ExecutionMode::PERFORMANCE),
         std::make_tuple(ov::hint::num_requests, 0),
         std::make_tuple(ov::hint::enable_cpu_pinning, false),
+        std::make_tuple(ov::hint::enable_cpu_reservation, false),
 
         std::make_tuple(ov::intel_gpu::hint::host_task_priority, ov::hint::Priority::MEDIUM),
         std::make_tuple(ov::intel_gpu::hint::queue_throttle, ov::intel_gpu::hint::ThrottleLevel::MEDIUM),
@@ -235,6 +236,11 @@ void ExecutionConfig::apply_user_properties(const cldnn::device_info& info) {
     if (info.supports_immad) {
         set_property(ov::intel_gpu::queue_type(QueueTypes::in_order));
     }
+    if (!is_set_by_user(ov::hint::enable_cpu_reservation)) {
+        if (get_property(ov::hint::enable_cpu_pinning)) {
+            set_property(ov::hint::enable_cpu_reservation(true));
+        }
+    }
 
     user_properties.clear();
 }

src/plugins/intel_gpu/tests/functional/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp

@@ -33,6 +33,7 @@ const std::vector<ov::AnyMap> gpu_compileModel_properties = {
     {ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY),
      ov::hint::num_requests(10),
      ov::hint::enable_cpu_pinning(true),
+     ov::hint::enable_cpu_reservation(false),
      ov::enable_profiling(true)}};
 
 INSTANTIATE_TEST_SUITE_P(smoke_gpuCompileModelBehaviorTests,