Add support for ZSTD compression for all LUT files (#1062)

* Add support for XZ compression for all LUTs file

* Replace XZ with ZSTD

* Update 3RD_PARTY_LICENSES

.gitmodules

@@ -13,9 +13,6 @@
 	path = external/qmqtt
 	url = https://github.com/emqx/qmqtt.git
 	ignore = dirty
-[submodule "external/xz"]
-	path = external/xz
-	url = https://github.com/tukaani-project/xz.git
 [submodule "external/mdns"]
 	path = external/mdns
 	url = https://github.com/mjansson/mdns.git
@@ -34,3 +31,6 @@
 [submodule "external/linalg"]
 	path = external/linalg
 	url = https://github.com/sgorsten/linalg.git
+[submodule "external/zstd"]
+	path = external/zstd
+	url = https://github.com/facebook/zstd.git

3RD_PARTY_LICENSES

@@ -3009,3 +3009,38 @@ INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHA
 FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 
 LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, 
 ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+==============================
+            zstd
+==============================
+
+BSD License
+
+For Zstandard software
+
+Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+ * Neither the name Facebook, nor Meta, nor the names of its contributors may
+   be used to endorse or promote products derived from this software without
+   specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

CMakeLists.txt

@@ -47,7 +47,7 @@ SET ( DEFAULT_BONJOUR                     ON  )
 SET ( DEFAULT_MQTT                        ON  )
 SET ( DEFAULT_STATIC_QT_PLUGINS           OFF )
 SET ( DEFAULT_PRECOMPILED_HEADERS         ON  )
-SET ( DEFAULT_XZ                          ON  )
+SET ( DEFAULT_ZSTD                        ON  )
 SET ( DEFAULT_POWER_MANAGEMENT            ON  )
 SET ( DEFAULT_SYSTRAY                     ON  )
 SET ( DEFAULT_SHARED_LIBS                 ON  )
@@ -382,26 +382,6 @@ if(DEFAULT_CEC)
 	endif()
 endif()
 
-# 7-zip as support
-if (WIN32)
-	# 7zip
-	cmake_policy(SET CMP0053 NEW)
-	set(MYENV "PROGRAMFILES(X86)")
-
-	find_program(SEVENZIP_BIN
-		NAMES 7z 7za
-		HINTS "${MINGWDIR}" "${MINGWLIBS}/bin" "$MYENV/7-zip" "$ENV{ProgramFiles}/7-zip" "$ENV{ProgramW6432}/7-zip"
-		PATH_SUFFIXES bin
-		DOC "7zip executable"
-	)
-
-	if (SEVENZIP_BIN)
-		message( STATUS "7-zip found: ${SEVENZIP_BIN}")
-	else()
-		message( FATAL_ERROR  "Please install 7-zip")
-	endif (SEVENZIP_BIN)
-endif()
-
 if (CMAKE_CROSSCOMPILING)
 	set ( DEFAULT_USE_SYSTEM_FLATBUFFERS_LIBS OFF )
 	SET ( DEFAULT_SHARED_LIBS                 OFF )
@@ -504,8 +484,8 @@ colorMe("ENABLE_PROTOBUF = " ${ENABLE_PROTOBUF})
 option(ENABLE_SYSTRAY "Enable SYSTRAY" ${DEFAULT_SYSTRAY})
 colorMe("ENABLE_SYSTRAY = " ${ENABLE_SYSTRAY})
 
-option(ENABLE_XZ "Enable XZ support" ${DEFAULT_XZ})
-colorMe("ENABLE_XZ = " ${ENABLE_XZ})
+option(ENABLE_ZSTD "Enable ZSTD support" ${DEFAULT_ZSTD})
+colorMe("ENABLE_ZSTD = " ${ENABLE_ZSTD})
 
 message( STATUS "\n${CyanColor}BUILD FEATURES${ColorReset}")
 
@@ -547,8 +527,8 @@ SET ( FLATBUFFERS_INSTALL_LIB_DIR ${CMAKE_BINARY_DIR}/flatbuf )
 
 find_package(GitVersion)
 
-if(ENABLE_XZ)
-	find_package(LibLZMA)
+if(ENABLE_ZSTD)
+	find_package(zstd)
 endif()
 
 # configure a header file to pass some of the CMake settings
@@ -694,13 +674,6 @@ if (WIN32)
 	endif()
 endif()
 
-# Add the source/lib directories
-add_subdirectory(external)
-add_subdirectory(sources)
-
-# Add resources directory
-add_subdirectory(resources)
-
 # remove generated files on make cleaan too
 LIST( APPEND GENERATED_QRC
 	${CMAKE_BINARY_DIR}/WebConfig.qrc
@@ -712,12 +685,19 @@ set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${GENERATED_QRC
 configure_file( "${OUR_CMAKE_MODULES}/cmake_uninstall.cmake.in" "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake" IMMEDIATE @ONLY)
 add_custom_target(uninstall COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake)
 
+# Add the source/lib directories
+add_subdirectory(external)
+add_subdirectory(sources)
+
+# Add resources directory
+add_subdirectory(resources)
+
 # enable make package - no code after this line !
 include (${CMAKE_CURRENT_SOURCE_DIR}/cmake/packages.cmake)
 
 # external targets
-if (WIN32 AND TARGET stb AND TARGET flatbuffers AND TARGET protobuf-nanopb AND TARGET lunasvg AND TARGET flatc AND TARGET qmqtt AND TARGET liblzma AND TARGET sqlite3 AND TARGET precompiled_hyperhdr_headers)
-	set_target_properties(stb qmqtt flatbuffers protobuf-nanopb lunasvg flatc resources uninstall liblzma sqlite3 precompiled_hyperhdr_headers PROPERTIES FOLDER ExternalLibsTargets)	
+if (WIN32 AND TARGET stb AND TARGET flatbuffers AND TARGET protobuf-nanopb AND TARGET lunasvg AND TARGET flatc AND TARGET qmqtt AND TARGET libzstd_static AND TARGET libzstd_shared AND TARGET zstd AND TARGET clean-all AND TARGET sqlite3 AND TARGET precompiled_hyperhdr_headers)
+	set_target_properties(stb qmqtt flatbuffers protobuf-nanopb lunasvg flatc resources uninstall libzstd_static libzstd_shared zstd clean-all sqlite3 precompiled_hyperhdr_headers PROPERTIES FOLDER ExternalLibsTargets)	
 else()
 	set_target_properties(resources uninstall PROPERTIES FOLDER ExternalLibsTargets)
 endif()

HyperhdrConfig.h.in

@@ -66,8 +66,8 @@
 // Define to enable system qmqtt
 #cmakedefine USE_SYSTEM_MQTT_LIBS
 
-// Define to enable protobuf
-#cmakedefine ENABLE_XZ
+// Define to enable zstd
+#cmakedefine ENABLE_ZSTD
 
 // Define to enable protobuf
 #cmakedefine ENABLE_PROTOBUF

cmake/cmake_modules/Findzstd.cmake

@@ -0,0 +1,29 @@
+# - Find zstd
+# Find the zstd compression library and includes
+#
+# ZSTD_INCLUDE_DIRS - where to find zstd.h, etc.
+# ZSTD_LIBRARIES - List of libraries when using zstd.
+# ZSTD_FOUND - True if zstd found.
+
+find_path(ZSTD_INCLUDE_DIRS
+  NAMES zstd.h
+  HINTS ${zstd_ROOT_DIR}/include)
+
+find_library(ZSTD_LIBRARIES
+  NAMES zstd
+  HINTS ${zstd_ROOT_DIR}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(zstd DEFAULT_MSG ZSTD_LIBRARIES ZSTD_INCLUDE_DIRS)
+
+mark_as_advanced(
+  ZSTD_LIBRARIES
+  ZSTD_INCLUDE_DIRS)
+
+if(ZSTD_FOUND AND NOT (TARGET zstd::zstd))
+  add_library (zstd::zstd UNKNOWN IMPORTED)
+  set_target_properties(zstd::zstd
+    PROPERTIES
+      IMPORTED_LOCATION ${ZSTD_LIBRARIES}
+      INTERFACE_INCLUDE_DIRECTORIES ${ZSTD_INCLUDE_DIRS})
+endif()

cmake/installers.cmake

@@ -20,9 +20,9 @@ macro(DeployApple TARGET)
 			install(CODE [[ file(INSTALL FILES $<TARGET_FILE:sqlite3> DESTINATION "${CMAKE_INSTALL_PREFIX}/hyperhdr.app/Contents/lib" TYPE SHARED_LIBRARY) ]] COMPONENT "HyperHDR")
 		endif()
 
-		# Copy utils-xz
-		if (USE_SHARED_LIBS AND TARGET utils-xz)
-			install(CODE [[ file(INSTALL FILES $<TARGET_FILE:utils-xz> DESTINATION "${CMAKE_INSTALL_PREFIX}/hyperhdr.app/Contents/lib" TYPE SHARED_LIBRARY) ]] COMPONENT "HyperHDR")
+		# Copy utils-zstd
+		if (USE_SHARED_LIBS AND TARGET utils-zstd)
+			install(CODE [[ file(INSTALL FILES $<TARGET_FILE:utils-zstd> DESTINATION "${CMAKE_INSTALL_PREFIX}/hyperhdr.app/Contents/lib" TYPE SHARED_LIBRARY) ]] COMPONENT "HyperHDR")
 		endif()
 
 		# Copy utils-image
@@ -243,9 +243,9 @@ macro(DeployUnix TARGET)
 			install(CODE [[ file(INSTALL FILES $<TARGET_FILE:utils-image> DESTINATION "${CMAKE_INSTALL_PREFIX}/share/hyperhdr/lib" TYPE SHARED_LIBRARY) ]] COMPONENT "HyperHDR")
 		endif()
 
-		# Copy UTILS-XZ lib
-		if (TARGET utils-image)
-			install(CODE [[ file(INSTALL FILES $<TARGET_FILE:utils-xz> DESTINATION "${CMAKE_INSTALL_PREFIX}/share/hyperhdr/lib" TYPE SHARED_LIBRARY) ]] COMPONENT "HyperHDR")
+		# Copy UTILS-ZSTD lib
+		if (TARGET utils-zstd)
+			install(CODE [[ file(INSTALL FILES $<TARGET_FILE:utils-zstd> DESTINATION "${CMAKE_INSTALL_PREFIX}/share/hyperhdr/lib" TYPE SHARED_LIBRARY) ]] COMPONENT "HyperHDR")
 		endif()
 
 
@@ -354,7 +354,7 @@ macro(DeployUnix TARGET)
 			"librt"
 			"libstdc++"
 			"libudev"
-			"libz"
+			"libz.so"
 			"libxrender1"
 			"libxi6"
 			"libxext6"
@@ -593,9 +593,9 @@ macro(DeployWindows TARGET)
 			install(CODE [[ file(INSTALL FILES $<TARGET_FILE:sqlite3> DESTINATION "${CMAKE_INSTALL_PREFIX}/bin" TYPE SHARED_LIBRARY) ]] COMPONENT "HyperHDR")
 		endif()
 
-		# Copy utils-xz
-		if (USE_SHARED_LIBS AND TARGET utils-xz)
-			install(CODE [[ file(INSTALL FILES $<TARGET_FILE:utils-xz> DESTINATION "${CMAKE_INSTALL_PREFIX}/bin" TYPE SHARED_LIBRARY) ]] COMPONENT "HyperHDR")
+		# Copy utils-zstd
+		if (USE_SHARED_LIBS AND TARGET utils-zstd)
+			install(CODE [[ file(INSTALL FILES $<TARGET_FILE:utils-zstd> DESTINATION "${CMAKE_INSTALL_PREFIX}/bin" TYPE SHARED_LIBRARY) ]] COMPONENT "HyperHDR")
 		endif()
 
 		# Copy utils-image

external/CMakeLists.txt

@@ -237,17 +237,22 @@ if ( ENABLE_MQTT )
 endif()
 
 #=============================================================================
-# XZ
+# ZSTD
 #=============================================================================
 
-if ( ENABLE_XZ )
-	if (NOT LIBLZMA_FOUND)
-		message( STATUS "System library xz could not be found. Using embedded xz library.")
-		option(BUILD_TESTING "" OFF)
-		add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/xz)
-		set_target_properties(liblzma PROPERTIES INTERFACE_INCLUDE_DIRECTORIES $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/external/xz/src/liblzma/api>)		
-		set_property(TARGET liblzma PROPERTY POSITION_INDEPENDENT_CODE ON)
-		add_library(LibLZMA::LibLZMA ALIAS liblzma)
+if ( ENABLE_ZSTD )
+	if (NOT zstd_FOUND)
+		message( STATUS "System library zstd could not be found. Using embedded zstd library.")
+		if(WIN32)
+			set(CMAKE_RC_FLAGS "${CMAKE_RC_FLAGS} -I${CMAKE_SOURCE_DIR}/external/zstd/lib")
+		endif()
+		set(ZSTD_BUILD_TESTS OFF)
+		set(ZSTD_LEGACY_SUPPORT OFF)
+		set(ZSTD_BUILD_PROGRAMS OFF)
+		add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/zstd/build/cmake)
+		set_target_properties(libzstd PROPERTIES INTERFACE_INCLUDE_DIRECTORIES $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/external/zstd/lib>)
+		set_property(TARGET libzstd PROPERTY POSITION_INDEPENDENT_CODE ON)
+		add_library(zstd::zstd ALIAS libzstd)
 	endif()
 ENDIF()
 

include/utils-zstd/utils-zstd.h

@@ -0,0 +1,10 @@
+#pragma once
+
+#ifndef _ZSTD_SHARED_API
+	#define _ZSTD_SHARED_API
+#endif
+
+#include <image/MemoryBuffer.h>
+
+_ZSTD_SHARED_API const char* DecompressZSTD(size_t downloadedDataSize, const uint8_t* downloadedData, const char* fileName);
+_ZSTD_SHARED_API const char* DecompressZSTD(size_t downloadedDataSize, const uint8_t* downloadedData, uint8_t* dest, int destSeek, int destSize);

include/utils/LutLoader.h

@@ -3,6 +3,7 @@
 #ifndef PCH_ENABLED
 	#include <QString>
 	#include <QList>
+	#include <QFile>
 #endif
 
 #include <utils/PixelFormat.h>
@@ -17,4 +18,7 @@ class LutLoader {
 		MemoryBuffer<uint8_t>	_lut;		
 
 		void loadLutFile(Logger* _log, PixelFormat color, const QList<QString>& files);
+	private:
+		bool decompressLut(Logger* _log, QFile& file, int index);
+		void hasher(int index, Logger* _log);
 };

sources/CMakeLists.txt

@@ -33,8 +33,8 @@ add_subdirectory(utils)
 add_subdirectory(utils-image)
 add_subdirectory(webserver)
 
-if(ENABLE_XZ)
-	add_subdirectory(utils-xz)
+if(ENABLE_ZSTD)
+	add_subdirectory(utils-zstd)
 endif()
 
 

sources/api/BaseAPI.cpp

@@ -31,8 +31,8 @@
 #include <base/GrabberHelper.h>
 #include <utils-image/utils-image.h>
 
-#ifdef ENABLE_XZ
-	#include <utils-xz/utils-xz.h>
+#ifdef ENABLE_ZSTD
+	#include <utils-zstd/utils-zstd.h>
 #endif
 
 #ifdef _WIN32
@@ -410,21 +410,21 @@ bool BaseAPI::saveSettings(const QJsonObject& data)
 
 QString BaseAPI::installLut(QNetworkReply* reply, QString fileName, int hardware_brightness, int hardware_contrast, int hardware_saturation, qint64 time)
 {
-#ifdef ENABLE_XZ
+#ifdef ENABLE_ZSTD
 	QString error = nullptr;
 
 	if (reply->error() == QNetworkReply::NetworkError::NoError)
 	{
 		QByteArray downloadedData = reply->readAll();
 
-		error = DecompressXZ(downloadedData.size(), reinterpret_cast<uint8_t*>(downloadedData.data()), QSTRING_CSTR(fileName));
+		error = DecompressZSTD(downloadedData.size(), reinterpret_cast<uint8_t*>(downloadedData.data()), QSTRING_CSTR(fileName));
 	}
 	else
 		error = "Could not download LUT file";
 
 	return error;
 #else
-	return "XZ support was disabled in the build configuration";
+	return "ZSTD support was disabled in the build configuration";
 #endif
 }
 

sources/api/HyperAPI.cpp

@@ -568,7 +568,7 @@ void HyperAPI::lutDownloaded(QNetworkReply* reply, int hardware_brightness, int
 
 void HyperAPI::handleLutInstallCommand(const QJsonObject& message, const QString& command, int tan)
 {
-	const QString& address = QString("%1/lut_lin_tables.3d.xz").arg(message["subcommand"].toString().trimmed());
+	const QString& address = QString("%1/lut_lin_tables.3d.zst").arg(message["subcommand"].toString().trimmed());
 	int hardware_brightness = message["hardware_brightness"].toInt(0);
 	int hardware_contrast = message["hardware_contrast"].toInt(0);
 	int hardware_saturation = message["hardware_saturation"].toInt(0);

sources/hyperhdr/CMakeLists.txt

@@ -54,12 +54,8 @@ else()
 		PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 endif()
 
-if(ENABLE_XZ)
-	if (LIBLZMA_FOUND)
-		target_link_libraries(hyperhdr LibLZMA::LibLZMA utils-xz)
-	else()
-		target_link_libraries(hyperhdr liblzma utils-xz)
-	endif()
+if(ENABLE_ZSTD)
+	target_link_libraries(hyperhdr zstd::zstd utils-zstd)
 endif()
 
 if (USE_STATIC_QT_PLUGINS)