From f5ccf62fda0f6aff3826a51107759f9192b4a61c Mon Sep 17 00:00:00 2001 From: MistEO Date: Tue, 19 Nov 2024 18:36:19 +0800 Subject: [PATCH 1/4] perf: ReadBinaryFromFile supports Chinese path --- fastdeploy/utils/utils.cc | 56 +++++++++++++++---- .../vision/ocr/ppocr/rec_postprocessor.cc | 6 +- 2 files changed, 51 insertions(+), 11 deletions(-) diff --git a/fastdeploy/utils/utils.cc b/fastdeploy/utils/utils.cc index c39b6adab8..080aa2e58c 100644 --- a/fastdeploy/utils/utils.cc +++ b/fastdeploy/utils/utils.cc @@ -15,6 +15,13 @@ #include "fastdeploy/utils/utils.h" #include +#include +#include +#include + +#ifdef _WIN32 +#include +#endif namespace fastdeploy { @@ -48,18 +55,47 @@ FDLogger& FDLogger::operator<<(std::ostream& (*os)(std::ostream&)) { return *this; } -bool ReadBinaryFromFile(const std::string& file, std::string* contents) { - std::ifstream fin(file, std::ios::in | std::ios::binary); - if (!fin.is_open()) { - FDERROR << "Failed to open file: " << file << " to read." << std::endl; +using os_string = std::filesystem::path::string_type; + +os_string to_osstring(std::string_view utf8_str) +{ +#ifdef _WIN32 + int len = MultiByteToWideChar(CP_UTF8, 0, utf8_str.data(), (int)utf8_str.size(), nullptr, 0); + os_string result(len, 0); + MultiByteToWideChar(CP_UTF8, 0, utf8_str.data(), (int)utf8_str.size(), result.data(), len); + return result; +#else + return std::string(utf8_str); +#endif +} + +bool ReadBinaryFromFile(const std::string& path, std::string* contents) +{ + if (!contents) { + return false; + } + std::ifstream file(to_osstring(path), std::ios::binary | std::ios::ate); + if (!file.is_open()) { return false; } - fin.seekg(0, std::ios::end); - contents->clear(); - contents->resize(fin.tellg()); - fin.seekg(0, std::ios::beg); - fin.read(&(contents->at(0)), contents->size()); - fin.close(); + auto& result = *contents; + result.clear(); + + auto fileSize = file.tellg(); + if (fileSize != -1) { + result.resize(fileSize); + file.seekg(0, std::ios::beg); + file.read(reinterpret_cast(result.data()), fileSize); + } + else { + // no size available, read to EOF + constexpr auto chunksize = 4096; + std::string chunk(chunksize, 0); + while (!file.fail()) { + file.read(reinterpret_cast(chunk.data()), chunksize); + result.insert(result.end(), chunk.data(), chunk.data() + file.gcount()); + } + } return true; } diff --git a/fastdeploy/vision/ocr/ppocr/rec_postprocessor.cc b/fastdeploy/vision/ocr/ppocr/rec_postprocessor.cc index d93c16907a..84402897e2 100644 --- a/fastdeploy/vision/ocr/ppocr/rec_postprocessor.cc +++ b/fastdeploy/vision/ocr/ppocr/rec_postprocessor.cc @@ -15,17 +15,21 @@ #include "fastdeploy/vision/ocr/ppocr/rec_postprocessor.h" #include "fastdeploy/utils/perf.h" #include "fastdeploy/vision/ocr/ppocr/utils/ocr_utils.h" +#include "fastdeploy/utils/utils.h" namespace fastdeploy { namespace vision { namespace ocr { std::vector ReadDict(const std::string& path) { - std::ifstream in(path); + std::string content; + ReadBinaryFromFile(path, &content); + std::stringstream in(std::move(content)); FDASSERT(in, "Cannot open file %s to read.", path.c_str()); std::string line; std::vector m_vec; while (getline(in, line)) { + if (!line.empty() && *line.rbegin() == '\r') line.pop_back(); m_vec.push_back(line); } m_vec.insert(m_vec.begin(), "#"); // blank char for ctc From 1bbba6fbe3cddb63de67e331ae48e1f51272b4f2 Mon Sep 17 00:00:00 2001 From: MistEO Date: Tue, 19 Nov 2024 19:11:30 +0800 Subject: [PATCH 2/4] fix: build error on old C++ std --- fastdeploy/utils/utils.cc | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/fastdeploy/utils/utils.cc b/fastdeploy/utils/utils.cc index 080aa2e58c..3dbef1212d 100644 --- a/fastdeploy/utils/utils.cc +++ b/fastdeploy/utils/utils.cc @@ -16,7 +16,6 @@ #include #include -#include #include #ifdef _WIN32 @@ -55,7 +54,12 @@ FDLogger& FDLogger::operator<<(std::ostream& (*os)(std::ostream&)) { return *this; } -using os_string = std::filesystem::path::string_type; +// using os_string = std::filesystem::path::string_type; +#ifdef _WIN32 +using os_string = std::wstring; +#else +using os_string = std::string; +#endif os_string to_osstring(std::string_view utf8_str) { @@ -74,12 +78,13 @@ bool ReadBinaryFromFile(const std::string& path, std::string* contents) if (!contents) { return false; } + auto& result = *contents; + result.clear(); + std::ifstream file(to_osstring(path), std::ios::binary | std::ios::ate); if (!file.is_open()) { return false; } - auto& result = *contents; - result.clear(); auto fileSize = file.tellg(); if (fileSize != -1) { From 5662fa39036d6cfd8310d49d966d9770344cff80 Mon Sep 17 00:00:00 2001 From: MistEO Date: Tue, 19 Nov 2024 19:16:16 +0800 Subject: [PATCH 3/4] fix: build error for CI --- fastdeploy/utils/utils.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fastdeploy/utils/utils.cc b/fastdeploy/utils/utils.cc index 3dbef1212d..13d78f8c79 100644 --- a/fastdeploy/utils/utils.cc +++ b/fastdeploy/utils/utils.cc @@ -90,14 +90,14 @@ bool ReadBinaryFromFile(const std::string& path, std::string* contents) if (fileSize != -1) { result.resize(fileSize); file.seekg(0, std::ios::beg); - file.read(reinterpret_cast(result.data()), fileSize); + file.read(const_cast(result.data()), fileSize); } else { // no size available, read to EOF constexpr auto chunksize = 4096; std::string chunk(chunksize, 0); while (!file.fail()) { - file.read(reinterpret_cast(chunk.data()), chunksize); + file.read(const_cast(chunk.data()), chunksize); result.insert(result.end(), chunk.data(), chunk.data() + file.gcount()); } } From e321ae55a1806752273af3cae9e670b95ac65a66 Mon Sep 17 00:00:00 2001 From: MistEO Date: Wed, 20 Nov 2024 00:27:27 +0800 Subject: [PATCH 4/4] fix: windows build error --- fastdeploy/utils/utils.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastdeploy/utils/utils.cc b/fastdeploy/utils/utils.cc index 13d78f8c79..5b95427d82 100644 --- a/fastdeploy/utils/utils.cc +++ b/fastdeploy/utils/utils.cc @@ -19,7 +19,7 @@ #include #ifdef _WIN32 -#include +#include #endif namespace fastdeploy {