diff --git a/Makefile b/Makefile index c5e07db8fa..96d50d81ba 100644 --- a/Makefile +++ b/Makefile @@ -37,8 +37,9 @@ tests: run_unit_tests run_hello_world run_hello_ovms run_hello_ovms: docker run $(OVMS_MEDIA_DOCKER_IMAGE):$(OVMS_MEDIA_IMAGE_TAG) bazel-bin/mediapipe/examples/desktop/hello_ovms/hello_ovms | grep -q "Output tensor data: 9 - 11" +MEDIAPIPE_UNSTABLE_TESTS_REGEX="MuxInputStreamHandlerTest.RemovesUnusedDataStreamPackets" run_unit_tests: - docker run -e http_proxy=$(HTTP_PROXY) -e https_proxy=$(HTTPS_PROXY) $(OVMS_MEDIA_DOCKER_IMAGE):$(OVMS_MEDIA_IMAGE_TAG) bazel test --define=MEDIAPIPE_DISABLE_GPU=1 //mediapipe/framework/... + docker run -e http_proxy=$(HTTP_PROXY) -e https_proxy=$(HTTPS_PROXY) $(OVMS_MEDIA_DOCKER_IMAGE):$(OVMS_MEDIA_IMAGE_TAG) bazel test --define=MEDIAPIPE_DISABLE_GPU=1 --test_output=streamed --test_filter="-${MEDIAPIPE_UNSTABLE_TESTS_REGEX}" //mediapipe/framework/... run_hello_world: docker run $(OVMS_MEDIA_DOCKER_IMAGE):$(OVMS_MEDIA_IMAGE_TAG) bazel-bin/mediapipe/examples/desktop/hello_world/hello_world diff --git a/mediapipe/calculators/ovms/BUILD b/mediapipe/calculators/ovms/BUILD index 0439bfe068..3516927040 100644 --- a/mediapipe/calculators/ovms/BUILD +++ b/mediapipe/calculators/ovms/BUILD @@ -17,38 +17,85 @@ licenses(["notice"]) package(default_visibility = ["//visibility:public"]) + cc_library( name = "ovms_calculator", - srcs = ["modelapiovmsadapter.cc", - "modelapiovmsadapter.hpp", - "openvinomodelserversessioncalculator.cc", - "openvinoinferencecalculator.cc"], + srcs = [ + ], deps = [ - "//mediapipe/calculators/ovms:openvinoinferencecalculator_cc_proto", - "//mediapipe/calculators/ovms:openvinomodelserversessioncalculator_cc_proto", "//mediapipe/calculators/openvino:openvino_tensors_to_classification_calculator_cc_proto", "//mediapipe/calculators/openvino:openvino_tensors_to_detections_calculator_cc_proto", "//mediapipe/calculators/openvino:openvino_converter_calculator_cc_proto", "//mediapipe/calculators/openvino:openvino_converter_calculator", "//mediapipe/calculators/openvino:openvino_tensors_to_classification_calculator", "//mediapipe/calculators/openvino:openvino_tensors_to_detections_calculator", - "//mediapipe/framework:calculator_framework", - "@org_tensorflow//tensorflow/core:framework", - "//mediapipe/framework/port:status", - "//mediapipe/framework/formats:tensor", # Tensor GetContract + ":modelapiovmsadapter", + ":openvinoinferencecalculator", + ":openvinomodelserversessioncalculator", + ], + copts = ["-Iexternal/ovms/src","-Isrc"], + linkopts = ["-Lmediapipe/"], + alwayslink = 1, +) + +cc_library( + name = "modelapiovmsadapter", + srcs = [ + "modelapiovmsadapter.cc" + ], + hdrs = [ + "modelapiovmsadapter.hpp" + ], + deps = [ + "//mediapipe/framework/port:logging", # TODO remove logs but need better error handling/reporting in Model API "@ovms//src:ovms_header", "@model_api//:model_api", - "@org_tensorflow//tensorflow/lite:framework_stable", # to use tflite + "@linux_openvino//:openvino", + ], + copts = ["-Iexternal/ovms/src","-Isrc"], +) + +cc_library( + name = "openvinoinferencecalculator", + srcs = [ + "openvinoinferencecalculator.cc" + ], + hdrs = [ + "openvinoinferencecalculator.h" + ], + deps = [ + ":modelapiovmsadapter", + ":openvinoinferencecalculator_cc_proto", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/formats:tensor", # Tensor GetContract + "@linux_openvino//:openvino", + "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/lite/c:c_api", ], copts = ["-Iexternal/ovms/src","-Isrc"], - linkopts = ["-Lmediapipe/"], + alwayslink = 1, +) + +cc_library( + name = "openvinomodelserversessioncalculator", + srcs = [ + "openvinomodelserversessioncalculator.cc" + ], + hdrs = [ + "openvinomodelserversessioncalculator.h" + ], + deps = [ + ":modelapiovmsadapter", + ":openvinomodelserversessioncalculator_cc_proto", + "//mediapipe/framework:calculator_framework", + "@linux_openvino//:openvino", + ], + copts = ["-Iexternal/ovms/src","-Isrc"], alwayslink = 1, ) load("@mediapipe//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library") -# ovms mediapipe_proto_library( name = "openvinoinferencecalculator_proto", srcs = ["openvinoinferencecalculator.proto"], @@ -67,3 +114,31 @@ mediapipe_proto_library( "@mediapipe//mediapipe/framework:calculator_proto", ], ) + +cc_test( + name = "openvinoinferencecalculator_test", + srcs = ["openvinoinferencecalculator_test.cc", + ], + deps = [ + ":ovms_calculator", + "@ovms//src:ovms_header", + "@ovms//src:ovms_lib", + "//mediapipe/framework/port:gtest_main", + "//mediapipe/framework/port:parse_text_proto", + ], + copts = ["-Iexternal/ovms/src","-Isrc"], +) + +cc_test( + name = "openvinomodelserversessioncalculator_test", + srcs = ["openvinomodelserversessioncalculator_test.cc", + ], + deps = [ + ":ovms_calculator", + "@ovms//src:ovms_header", + "@ovms//src:ovms_lib", + "//mediapipe/framework/port:gtest_main", + "//mediapipe/framework/port:parse_text_proto", + ], + copts = ["-Iexternal/ovms/src","-Isrc"], +) diff --git a/mediapipe/calculators/ovms/modelapiovmsadapter.cc b/mediapipe/calculators/ovms/modelapiovmsadapter.cc index b825b03ccd..1460771613 100644 --- a/mediapipe/calculators/ovms/modelapiovmsadapter.cc +++ b/mediapipe/calculators/ovms/modelapiovmsadapter.cc @@ -25,10 +25,10 @@ #include +#include "ovms.h" // NOLINT #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/canonical_errors.h" +#include "mediapipe/framework/port/logging.h" #pragma GCC diagnostic pop // here we need to decide if we have several calculators (1 for OVMS repository, 1-N inside mediapipe) // for the one inside OVMS repo it makes sense to reuse code from ovms lib diff --git a/mediapipe/calculators/ovms/modelapiovmsadapter.hpp b/mediapipe/calculators/ovms/modelapiovmsadapter.hpp index dd1c71d78a..3c677324ff 100644 --- a/mediapipe/calculators/ovms/modelapiovmsadapter.hpp +++ b/mediapipe/calculators/ovms/modelapiovmsadapter.hpp @@ -26,15 +26,11 @@ #include // TODO fix path model_api/model_api/cpp/adapters/include/adapters/inference_adapter.h #include -#include "ovms.h" // NOLINT -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/canonical_errors.h" -#pragma GCC diagnostic pop // here we need to decide if we have several calculators (1 for OVMS repository, 1-N inside mediapipe) // for the one inside OVMS repo it makes sense to reuse code from ovms lib +class OVMS_Server_; +typedef struct OVMS_Server_ OVMS_Server; namespace mediapipe { namespace ovms { diff --git a/mediapipe/calculators/ovms/openvinoinferencecalculator.cc b/mediapipe/calculators/ovms/openvinoinferencecalculator.cc index cb562225ee..6e52aa302a 100644 --- a/mediapipe/calculators/ovms/openvinoinferencecalculator.cc +++ b/mediapipe/calculators/ovms/openvinoinferencecalculator.cc @@ -338,6 +338,7 @@ class OpenVINOInferenceCalculator : public CalculatorBase { LOG(INFO) << "OpenVINOInferenceCalculator GetContract start"; RET_CHECK(!cc->Inputs().GetTags().empty()); RET_CHECK(!cc->Outputs().GetTags().empty()); + RET_CHECK(cc->InputSidePackets().HasTag(SESSION_TAG)); for (const std::string& tag : cc->Inputs().GetTags()) { // could be replaced with absl::StartsWith when migrated to MP if (startsWith(tag, OVTENSORS_TAG)) { @@ -515,10 +516,10 @@ class OpenVINOInferenceCalculator : public CalculatorBase { try { output = session->infer(input); } catch (const std::exception& e) { - LOG(INFO) << "Catched exception from session infer():" << e.what(); + LOG(INFO) << "Caught exception from session infer():" << e.what(); RET_CHECK(false); } catch (...) { - LOG(INFO) << "Catched unknown exception from session infer()"; + LOG(INFO) << "Caught unknown exception from session infer()"; RET_CHECK(false); } auto outputsCount = output.size(); @@ -659,7 +660,6 @@ class OpenVINOInferenceCalculator : public CalculatorBase { LOG(INFO) << "Failed to deserialize tensor error:" << e.what(); RET_CHECK(false); } - LOG(INFO) << "OVMS calculator will process TfLite tensors"; } LOG(INFO) << "OpenVINOInferenceCalculator process end"; return absl::OkStatus(); diff --git a/mediapipe/calculators/ovms/openvinoinferencecalculator.h b/mediapipe/calculators/ovms/openvinoinferencecalculator.h new file mode 100644 index 0000000000..8a92b8120c --- /dev/null +++ b/mediapipe/calculators/ovms/openvinoinferencecalculator.h @@ -0,0 +1,44 @@ +#pragma once +//***************************************************************************** +// Copyright 2023 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/port/canonical_errors.h" +#pragma GCC diagnostic pop +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow/lite/interpreter.h" +#pragma GCC diagnostic pop +class InferenceAdapter; +namespace mediapipe { +class OpenVINOInferenceCalculator : public CalculatorBase { + std::shared_ptr<::InferenceAdapter> session{nullptr}; + std::unordered_map outputNameToTag; + std::vector input_order_list; + std::vector output_order_list; + std::unique_ptr interpreter_ = absl::make_unique(); + bool initialized = false; +public: + static absl::Status GetContract(CalculatorContract* cc); + absl::Status Close(CalculatorContext* cc) override final; + absl::Status Open(CalculatorContext* cc) override final; + absl::Status Process(CalculatorContext* cc) override final; +}; +} // namespace mediapipe diff --git a/mediapipe/calculators/ovms/openvinoinferencecalculator_test.cc b/mediapipe/calculators/ovms/openvinoinferencecalculator_test.cc new file mode 100644 index 0000000000..6e0d857ccf --- /dev/null +++ b/mediapipe/calculators/ovms/openvinoinferencecalculator_test.cc @@ -0,0 +1,179 @@ +//***************************************************************************** +// Copyright 2023 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "openvinoinferencecalculator.h" +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "ovms.h" // NOLINT +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#include "mediapipe/calculators/ovms/openvinoinferencecalculator.pb.h" +#include "mediapipe/framework/port/canonical_errors.h" +#include "mediapipe/framework/deps/status_matchers.h" +#include "mediapipe/framework/formats/tensor.h" +#include "mediapipe/framework/port/canonical_errors.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/parse_text_proto.h" +#include "tensorflow/core/framework/tensor.h" +#pragma GCC diagnostic pop +using mediapipe::Adopt; +using mediapipe::CalculatorContract; +using mediapipe::CalculatorGraph; +using mediapipe::CalculatorGraphConfig; +using mediapipe::OpenVINOInferenceCalculator; +using mediapipe::CalculatorState; +using mediapipe::ParseTextProtoOrDie; +using mediapipe::Packet; +using mediapipe::PacketType; +using mediapipe::Timestamp; + +class OpenVINOInferenceCalculatorTest : public ::testing::Test { +PacketType OVTENSOR_TYPE; +PacketType OVTENSORS_TYPE; +PacketType MPTENSOR_TYPE; +PacketType MPTENSORS_TYPE; +public: + void SetUp() override { + OVTENSOR_TYPE.Set(); + OVTENSORS_TYPE.Set>(); + MPTENSOR_TYPE.Set(); + MPTENSORS_TYPE.Set>(); + } +}; + +TEST_F(OpenVINOInferenceCalculatorTest, VerifySupportedTags) { + auto calculator = + mediapipe::ParseTextProtoOrDie( + R"pb( + calculator: "OpenVINOInferenceCalculator" + input_side_packet: "SESSION:not_used_session" + input_stream: "OVTENSOR:input" + input_stream: "MPTENSOR:input" + input_stream: "TFTENSOR:input" + input_stream: "TFLITE_TENSOR:input" + output_stream: "OVTENSORS:output" + output_stream: "MPTENSORS:output" + output_stream: "TFTENSORS:output" + output_stream: "TFLITE_TENSORS:output" + )pb"); + auto cc = absl::make_unique(); + cc->Initialize(calculator); + auto abslStatus = mediapipe::OpenVINOInferenceCalculator::GetContract(cc.get()); + ASSERT_EQ(abslStatus.code(), absl::StatusCode::kOk) << abslStatus.message(); + EXPECT_EQ(1, cc->InputSidePackets().TagMap()->NumEntries()); + EXPECT_EQ(0, cc->OutputSidePackets().NumEntries()); + auto& inputPacketsTags = cc->Inputs(); + auto& outputPacketsTags = cc->Outputs(); + EXPECT_EQ(4, inputPacketsTags.TagMap()->NumEntries()); + EXPECT_EQ(4, outputPacketsTags.TagMap()->NumEntries()); +} +TEST_F(OpenVINOInferenceCalculatorTest, VerifyNotAllowedEmptySideInputPacket) { + auto calculator = + mediapipe::ParseTextProtoOrDie( + R"pb( + calculator: "OpenVINOInferenceCalculator" + input_stream: "OVTENSOR:input" + output_stream: "OVTENSOR:output" + )pb"); + auto cc = absl::make_unique(); + cc->Initialize(calculator); + auto abslStatus = mediapipe::OpenVINOInferenceCalculator::GetContract(cc.get()); + EXPECT_EQ(abslStatus.code(), absl::StatusCode::kInternal) << abslStatus.message(); +} +TEST_F(OpenVINOInferenceCalculatorTest, VerifyNotAllowedSideOutputPacket) { + auto calculator = + mediapipe::ParseTextProtoOrDie( + R"pb( + calculator: "OpenVINOInferenceCalculator" + output_side_packet: "SESSION:not_used_session" + input_stream: "OVTENSOR:input" + output_stream: "OVTENSOR:output" + )pb"); + auto cc = absl::make_unique(); + cc->Initialize(calculator); + auto abslStatus = mediapipe::OpenVINOInferenceCalculator::GetContract(cc.get()); + EXPECT_EQ(abslStatus.code(), absl::StatusCode::kInternal) << abslStatus.message(); +} +TEST_F(OpenVINOInferenceCalculatorTest, BasicDummyInference) { + std::string graph_proto = R"( + input_stream: "input" + output_stream: "output" + node { + calculator: "OpenVINOModelServerSessionCalculator" + output_side_packet: "SESSION:session" + node_options: { + [type.googleapis.com / mediapipe.OpenVINOModelServerSessionCalculatorOptions]: { + servable_name: "dummy" + server_config: "/mediapipe/mediapipe/calculators/ovms/test_data/config.json" + } + } + } + node { + calculator: "OpenVINOInferenceCalculator" + input_side_packet: "SESSION:session" + input_stream: "OVTENSOR:input" + output_stream: "OVTENSOR:output" + node_options: { + [type.googleapis.com / mediapipe.OpenVINOInferenceCalculatorOptions]: { + output_order_list: ["Identity:0", "Identity_1:0"] + tag_to_input_tensor_names { + key: "OVTENSOR" + value: "b" + } + tag_to_output_tensor_names { + key: "OVTENSOR" + value: "a" + } + } + } + } + )"; + CalculatorGraphConfig graph_config = + ParseTextProtoOrDie(graph_proto); + const std::string inputStreamName = "input"; + const std::string outputStreamName = "output"; + // avoid creating pollers, retreiving packets etc. + std::vector output_packets; + mediapipe::tool::AddVectorSink(outputStreamName, &graph_config, &output_packets); + CalculatorGraph graph(graph_config); + MP_ASSERT_OK(graph.StartRun({})); + auto datatype = ov::element::Type_t::f32; + ov::Shape shape{1,10}; + std::vector data{0,1,2,3,4,5,6,7,8,9}; + auto inputTensor = std::make_unique(datatype, shape, data.data()); + MP_ASSERT_OK(graph.AddPacketToInputStream( + inputStreamName, Adopt(inputTensor.release()).At(Timestamp(0)))); + MP_ASSERT_OK(graph.CloseInputStream(inputStreamName)); + MP_ASSERT_OK(graph.WaitUntilIdle()); + ASSERT_EQ(1, output_packets.size()); + const ov::Tensor& outputTensor = + output_packets[0].Get(); + MP_ASSERT_OK(graph.WaitUntilDone()); + EXPECT_EQ(datatype, outputTensor.get_element_type()); + EXPECT_THAT(outputTensor.get_shape(), testing::ElementsAre(1,10)); + const void* outputData = outputTensor.data(); + for (size_t i = 0; i < data.size(); ++i) { + EXPECT_EQ(data[i] + 1, *(reinterpret_cast(outputData) + i)) << i; + } +} diff --git a/mediapipe/calculators/ovms/openvinomodelserversessioncalculator.cc b/mediapipe/calculators/ovms/openvinomodelserversessioncalculator.cc index 94a8803055..004025915b 100644 --- a/mediapipe/calculators/ovms/openvinomodelserversessioncalculator.cc +++ b/mediapipe/calculators/ovms/openvinomodelserversessioncalculator.cc @@ -13,6 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. //***************************************************************************** +#include "openvinomodelserversessioncalculator.h" #include #include #include @@ -26,17 +27,13 @@ #include "ovms.h" // NOLINT #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#include "mediapipe/calculators/ovms/openvinomodelserversessioncalculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/port/canonical_errors.h" #include "modelapiovmsadapter.hpp" -#include "mediapipe/calculators/ovms/openvinomodelserversessioncalculator.pb.h" #pragma GCC diagnostic pop -// here we need to decide if we have several calculators (1 for OVMS repository, 1-N inside mediapipe) -// for the one inside OVMS repo it makes sense to reuse code from ovms lib namespace mediapipe { - using ovms::OVMSInferenceAdapter; -using std::endl; const std::string SESSION_TAG{"SESSION"}; ov::Core UNUSED_OV_CORE; @@ -117,107 +114,94 @@ class SettingsGuard { } }; -class OpenVINOModelServerSessionCalculator : public CalculatorBase { - std::shared_ptr<::InferenceAdapter> adapter; - std::unordered_map outputNameToTag; - // TODO where to place members - OVMS_Server* cserver{nullptr}; - static bool triedToStartOVMS; - static std::mutex loadingMtx; -public: - static absl::Status GetContract(CalculatorContract* cc) { - LOG(INFO) << "OpenVINOModelServerSessionCalculator GetContract start"; - RET_CHECK(cc->Inputs().GetTags().empty()); - RET_CHECK(cc->Outputs().GetTags().empty()); - cc->OutputSidePackets().Tag(SESSION_TAG.c_str()).Set>(); - const auto& options = cc->Options(); - RET_CHECK(!options.servable_name().empty()); - // TODO validate version from string - // TODO validate service url format - // this is for later support for remote server inference - LOG(INFO) << "OpenVINOModelServerSessionCalculator GetContract end"; - return absl::OkStatus(); - } +absl::Status OpenVINOModelServerSessionCalculator::GetContract(CalculatorContract* cc) { + LOG(INFO) << "OpenVINOModelServerSessionCalculator GetContract start"; + RET_CHECK(cc->Inputs().GetTags().empty()); + RET_CHECK(cc->Outputs().GetTags().empty()); + cc->OutputSidePackets().Tag(SESSION_TAG.c_str()).Set>(); + const auto& options = cc->Options(); + RET_CHECK(!options.servable_name().empty()); + LOG(INFO) << "OpenVINOModelServerSessionCalculator GetContract end"; + return absl::OkStatus(); +} - absl::Status Close(CalculatorContext* cc) final { - LOG(INFO) << "OpenVINOModelServerSessionCalculator Close"; - return absl::OkStatus(); - } - absl::Status Open(CalculatorContext* cc) final { - LOG(INFO) << "OpenVINOModelServerSessionCalculator Open start"; - for (CollectionItemId id = cc->Inputs().BeginId(); - id < cc->Inputs().EndId(); ++id) { - if (!cc->Inputs().Get(id).Header().IsEmpty()) { - cc->Outputs().Get(id).SetHeader(cc->Inputs().Get(id).Header()); - } - } - if (cc->OutputSidePackets().NumEntries() != 0) { - for (CollectionItemId id = cc->InputSidePackets().BeginId(); - id < cc->InputSidePackets().EndId(); ++id) { - cc->OutputSidePackets().Get(id).Set(cc->InputSidePackets().Get(id)); - } +absl::Status OpenVINOModelServerSessionCalculator::Close(CalculatorContext* cc) { + LOG(INFO) << "OpenVINOModelServerSessionCalculator Close"; + return absl::OkStatus(); +} +absl::Status OpenVINOModelServerSessionCalculator::Open(CalculatorContext* cc) { + LOG(INFO) << "OpenVINOModelServerSessionCalculator Open start"; + for (CollectionItemId id = cc->Inputs().BeginId(); + id < cc->Inputs().EndId(); ++id) { + if (!cc->Inputs().Get(id).Header().IsEmpty()) { + cc->Outputs().Get(id).SetHeader(cc->Inputs().Get(id).Header()); } - cc->SetOffset(TimestampDiff(0)); - - const auto& options = cc->Options(); - // if config is in calc then we start the server - LOG(INFO) << "Will check if we want to start server"; - if (!options.server_config().empty()) { - // Lock access to server from multiple calculator instances during the model loading phase - std::unique_lock lk(OpenVINOModelServerSessionCalculator::loadingMtx); - bool isServerReady = false; - bool isServerLive = false; - OVMS_ServerNew(&cserver); - - ASSERT_CAPI_STATUS_NULL(OVMS_ServerLive(cserver, &isServerLive)); - - if (triedToStartOVMS) { - RET_CHECK(isServerLive); - } else if (!isServerLive) { - LOG(INFO) << "Will start new server"; - triedToStartOVMS = true; - SettingsGuard guard; - OVMS_ServerSettingsNew(&guard.serverSettings); - OVMS_ModelsSettingsNew(&guard.modelsSettings); - OVMS_ModelsSettingsSetConfigPath(guard.modelsSettings, options.server_config().c_str()); - LOG(INFO) << "state config file:" << options.server_config(); - OVMS_ServerSettingsSetLogLevel(guard.serverSettings, OVMS_LOG_DEBUG); - - ASSERT_CAPI_STATUS_NULL(OVMS_ServerStartFromConfigurationFile(cserver, guard.serverSettings, guard.modelsSettings)); - - ASSERT_CAPI_STATUS_NULL(OVMS_ServerReady(cserver, &isServerReady)); - RET_CHECK(isServerReady); - LOG(INFO) << "Server started"; - } + } + if (cc->OutputSidePackets().NumEntries() != 0) { + for (CollectionItemId id = cc->InputSidePackets().BeginId(); + id < cc->InputSidePackets().EndId(); ++id) { + cc->OutputSidePackets().Get(id).Set(cc->InputSidePackets().Get(id)); } - - const std::string& servableName = options.servable_name(); - const std::string& servableVersionStr = options.servable_version(); - auto servableVersionOpt = stou32(servableVersionStr); - // 0 means default - uint32_t servableVersion = servableVersionOpt.value_or(0); - auto session = std::make_shared(servableName, servableVersion); - try { - session->loadModel(nullptr, UNUSED_OV_CORE, "UNUSED", {}); - } catch (const std::exception& e) { - LOG(INFO) << "Catched exception with message: " << e.what(); - RET_CHECK(false); - } catch (...) { - LOG(INFO) << "Catched unknown exception"; - RET_CHECK(false); + } + cc->SetOffset(TimestampDiff(0)); + + const auto& options = cc->Options(); + // if config is in calc then we start the server + LOG(INFO) << "Will check if we want to start server"; + if (!options.server_config().empty()) { + // Lock access to server from multiple calculator instances during the model loading phase + std::unique_lock lk(OpenVINOModelServerSessionCalculator::loadingMtx); + bool isServerReady = false; + bool isServerLive = false; + OVMS_ServerNew(&cserver); + + ASSERT_CAPI_STATUS_NULL(OVMS_ServerLive(cserver, &isServerLive)); + + if (triedToStartOVMS) { + RET_CHECK(isServerLive); + } else if (!isServerLive) { + LOG(INFO) << "Will start new server"; + triedToStartOVMS = true; + SettingsGuard guard; + OVMS_ServerSettingsNew(&guard.serverSettings); + OVMS_ModelsSettingsNew(&guard.modelsSettings); + OVMS_ModelsSettingsSetConfigPath(guard.modelsSettings, options.server_config().c_str()); + LOG(INFO) << "state config file:" << options.server_config(); + OVMS_ServerSettingsSetLogLevel(guard.serverSettings, OVMS_LOG_DEBUG); + + ASSERT_CAPI_STATUS_NULL(OVMS_ServerStartFromConfigurationFile(cserver, guard.serverSettings, guard.modelsSettings)); + + ASSERT_CAPI_STATUS_NULL(OVMS_ServerReady(cserver, &isServerReady)); + RET_CHECK(isServerReady); + LOG(INFO) << "Server started"; } - - LOG(INFO) << "OpenVINOModelServerSessionCalculator create adapter"; - cc->OutputSidePackets().Tag(SESSION_TAG.c_str()).Set(MakePacket>(session)); - LOG(INFO) << "OpenVINOModelServerSessionCalculator Open end"; - return absl::OkStatus(); } - - absl::Status Process(CalculatorContext* cc) final { - LOG(INFO) << "OpenVINOModelServerSessionCalculator Process"; - return absl::OkStatus(); + const std::string& servableName = options.servable_name(); + const std::string& servableVersionStr = options.servable_version(); + auto servableVersionOpt = stou32(servableVersionStr); + // 0 means default + uint32_t servableVersion = servableVersionOpt.value_or(0); + auto session = std::make_shared(servableName, servableVersion); + try { + session->loadModel(nullptr, UNUSED_OV_CORE, "UNUSED", {}); + } catch (const std::exception& e) { + LOG(INFO) << "Caught exception with message: " << e.what(); + RET_CHECK(false); + } catch (...) { + LOG(INFO) << "Caught unknown exception"; + RET_CHECK(false); } -}; + + LOG(INFO) << "OpenVINOModelServerSessionCalculator create adapter"; + cc->OutputSidePackets().Tag(SESSION_TAG.c_str()).Set(MakePacket>(session)); + LOG(INFO) << "OpenVINOModelServerSessionCalculator Open end"; + return absl::OkStatus(); +} + +absl::Status OpenVINOModelServerSessionCalculator::Process(CalculatorContext* cc) { + LOG(INFO) << "OpenVINOModelServerSessionCalculator Process"; + return absl::OkStatus(); +} bool OpenVINOModelServerSessionCalculator::triedToStartOVMS = false; std::mutex OpenVINOModelServerSessionCalculator::loadingMtx; diff --git a/mediapipe/calculators/ovms/openvinomodelserversessioncalculator.h b/mediapipe/calculators/ovms/openvinomodelserversessioncalculator.h new file mode 100644 index 0000000000..e21a58eb23 --- /dev/null +++ b/mediapipe/calculators/ovms/openvinomodelserversessioncalculator.h @@ -0,0 +1,51 @@ +//***************************************************************************** +// Copyright 2023 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ovms.h" // NOLINT +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/port/canonical_errors.h" +#include "modelapiovmsadapter.hpp" +#include "mediapipe/calculators/ovms/openvinomodelserversessioncalculator.pb.h" +#pragma GCC diagnostic pop +namespace mediapipe { + +using ovms::OVMSInferenceAdapter; + +class OpenVINOModelServerSessionCalculator : public CalculatorBase { + std::shared_ptr<::InferenceAdapter> adapter; + std::unordered_map outputNameToTag; + OVMS_Server* cserver{nullptr}; + static bool triedToStartOVMS; + static std::mutex loadingMtx; +public: + static absl::Status GetContract(CalculatorContract* cc); + absl::Status Close(CalculatorContext* cc) override final; + absl::Status Open(CalculatorContext* cc) override final; + + absl::Status Process(CalculatorContext* cc) override final; +}; +} // namespace mediapipe diff --git a/mediapipe/calculators/ovms/openvinomodelserversessioncalculator.proto b/mediapipe/calculators/ovms/openvinomodelserversessioncalculator.proto index d4ff159eb8..eb63b15df0 100644 --- a/mediapipe/calculators/ovms/openvinomodelserversessioncalculator.proto +++ b/mediapipe/calculators/ovms/openvinomodelserversessioncalculator.proto @@ -26,7 +26,7 @@ message OpenVINOModelServerSessionCalculatorOptions { optional OpenVINOModelServerSessionCalculatorOptions ext = 113473744; } required string servable_name = 1; - required string servable_version = 2; + optional string servable_version = 2; // service_url: "13.21.212.171:9718" optional string service_url = 3; optional string server_config = 4; diff --git a/mediapipe/calculators/ovms/openvinomodelserversessioncalculator_test.cc b/mediapipe/calculators/ovms/openvinomodelserversessioncalculator_test.cc new file mode 100644 index 0000000000..38955124e2 --- /dev/null +++ b/mediapipe/calculators/ovms/openvinomodelserversessioncalculator_test.cc @@ -0,0 +1,153 @@ +//***************************************************************************** +// Copyright 2023 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "openvinomodelserversessioncalculator.h" +#include +#include +#include +#include +#include +#include + +#include "mediapipe/framework/port/gtest.h" + +#include // TODO fix path model_api/model_api/cpp/adapters/include/adapters/inference_adapter.h +#include +#include + +#include "ovms.h" // NOLINT +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#include "mediapipe/calculators/ovms/openvinomodelserversessioncalculator.pb.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/port/canonical_errors.h" +#include "mediapipe/framework/deps/status_matchers.h" +//#include "mediapipe/framework/calculator_runner.h" +#include "mediapipe/framework/formats/tensor.h" +#include "mediapipe/framework/port/canonical_errors.h" +#include "mediapipe/framework/port/parse_text_proto.h" +#include "mediapipe/framework/tool/tag_map_helper.h" +#pragma GCC diagnostic pop +// here we need to decide if we have several calculators (1 for OVMS repository, 1-N inside mediapipe) +// for the one inside OVMS repo it makes sense to reuse code from ovms lib +using mediapipe::CalculatorContract; +using mediapipe::OpenVINOModelServerSessionCalculator; +using mediapipe::CalculatorState; +using mediapipe::ParseTextProtoOrDie; +using mediapipe::PacketType; + +class OpenVINOModelServerSessionCalculatorTest : public ::testing::Test { +PacketType OVTENSOR_TYPE; +PacketType OVTENSORS_TYPE; +PacketType MPTENSOR_TYPE; +PacketType MPTENSORS_TYPE; +public: + void SetUp() override { + OVTENSOR_TYPE.Set(); + OVTENSORS_TYPE.Set>(); + MPTENSOR_TYPE.Set(); + MPTENSORS_TYPE.Set>(); + } +}; + +TEST_F(OpenVINOModelServerSessionCalculatorTest, VerifyCorrectPbtxtWithAllOptions) { + auto node = + mediapipe::ParseTextProtoOrDie( + R"pb( + calculator: "OpenVINOModelServerSessionCalculator" + output_side_packet: "SESSION:session" + node_options: { + [type.googleapis.com / mediapipe.OpenVINOModelServerSessionCalculatorOptions]: { + servable_name: "not_used_name" + servable_version: "1" + server_config: "mediapipe/config.json" + service_url: "192.168.0.1:9178" + } + } + )pb"); + auto cc = absl::make_unique(); + cc->Initialize(node); + auto abslStatus = mediapipe::OpenVINOModelServerSessionCalculator::GetContract(cc.get()); + ASSERT_EQ(abslStatus.code(), absl::StatusCode::kOk) << abslStatus.message(); + EXPECT_EQ(0, cc->InputSidePackets().TagMap()->NumEntries()); + EXPECT_EQ(1, cc->OutputSidePackets().NumEntries()); + auto& inputPacketsTags = cc->Inputs(); + auto& outputPacketsTags = cc->Outputs(); + EXPECT_EQ(0, inputPacketsTags.TagMap()->NumEntries()); + EXPECT_EQ(0, outputPacketsTags.TagMap()->NumEntries()); +} +TEST_F(OpenVINOModelServerSessionCalculatorTest, VerifyOptionalityOfOptionFields) { + // servable_version, server_config, server_url are optional + auto node = + mediapipe::ParseTextProtoOrDie( + R"pb( + calculator: "OpenVINOModelServerSessionCalculator" + output_side_packet: "SESSION:session" + node_options: { + [type.googleapis.com / mediapipe.OpenVINOModelServerSessionCalculatorOptions]: { + servable_name: "not_used_name" + } + } + )pb"); + auto cc = absl::make_unique(); + cc->Initialize(node); + auto abslStatus = mediapipe::OpenVINOModelServerSessionCalculator::GetContract(cc.get()); + EXPECT_EQ(abslStatus.code(), absl::StatusCode::kOk) << abslStatus.message(); +} +TEST_F(OpenVINOModelServerSessionCalculatorTest, VerifyMandatorityOfFields) { + // servable_version, server_config, server_url are optional + mediapipe::CalculatorGraphConfig::Node node; + bool success = mediapipe::ParseTextProto( + R"pb( + calculator: "OpenVINOModelServerSessionCalculator" + output_side_packet: "SESSION:session" + node_options: { + [type.googleapis.com / mediapipe.OpenVINOModelServerSessionCalculatorOptions]: { + # commented out servable_name: "not_used_name" + servable_version: "1" + server_config: "mediapipe/config.json" + service_url: "192.168.0.1:9178" + } + } + )pb", &node); + EXPECT_FALSE(success); +} +TEST_F(OpenVINOModelServerSessionCalculatorTest, VerifyNonExistingFields) { + mediapipe::CalculatorGraphConfig::Node node; + bool success = mediapipe::ParseTextProto( + R"pb( + calculator: "OpenVINOModelServerSessionCalculator" + output_side_packet: "SESSION:session" + node_options: { + [type.googleapis.com / mediapipe.OpenVINOModelServerSessionCalculatorOptions]: { + servable_name: "not_used_name" + some_random_name: 1 + } + } + )pb", &node); + EXPECT_FALSE(success); +} +TEST_F(OpenVINOModelServerSessionCalculatorTest, MissingAllOptions) { + auto node = + mediapipe::ParseTextProtoOrDie( + R"pb( + calculator: "OpenVINOModelServerSessionCalculator" + output_side_packet: "SESSION:session" + )pb"); + auto cc = absl::make_unique(); + cc->Initialize(node); + auto abslStatus = mediapipe::OpenVINOModelServerSessionCalculator::GetContract(cc.get()); + ASSERT_EQ(abslStatus.code(), absl::StatusCode::kInternal) << abslStatus.message(); +} diff --git a/mediapipe/calculators/ovms/test_data/config.json b/mediapipe/calculators/ovms/test_data/config.json new file mode 100644 index 0000000000..9c76c669e5 --- /dev/null +++ b/mediapipe/calculators/ovms/test_data/config.json @@ -0,0 +1,10 @@ +{ + "model_config_list" : [ + { + "config": { + "name": "dummy", + "base_path": "./dummy" + } + } + ] +} diff --git a/mediapipe/calculators/ovms/test_data/dummy/1/dummy.bin b/mediapipe/calculators/ovms/test_data/dummy/1/dummy.bin new file mode 100644 index 0000000000..9c12e8adf9 Binary files /dev/null and b/mediapipe/calculators/ovms/test_data/dummy/1/dummy.bin differ diff --git a/mediapipe/calculators/ovms/test_data/dummy/1/dummy.xml b/mediapipe/calculators/ovms/test_data/dummy/1/dummy.xml new file mode 100644 index 0000000000..8df1493123 --- /dev/null +++ b/mediapipe/calculators/ovms/test_data/dummy/1/dummy.xml @@ -0,0 +1,99 @@ + + + + + + + + 1 + 10 + + + + + + + + 1 + 1 + + + + + + + 1 + 10 + + + 1 + 1 + + + + + 1 + 10 + + + + + + + 1 + 10 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +