Skip to content

Commit

Permalink
Add support for XZ compression for all LUTs file
Browse files Browse the repository at this point in the history
  • Loading branch information
@awawa-dev
awawa-dev committed Jan 22, 2025
1 parent 97b138d commit a47d436
Show file tree
Hide file tree
Showing 4 changed files with 184 additions and 12 deletions.
3 changes: 3 additions & 0 deletions include/utils-xz/utils-xz.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,7 @@
#define _XZ_SHARED_API
#endif

#include <image/MemoryBuffer.h>

_XZ_SHARED_API const char* DecompressXZ(size_t downloadedDataSize, const uint8_t* downloadedData, const char* fileName);
_XZ_SHARED_API const char* DecompressXZ(size_t downloadedDataSize, const uint8_t* downloadedData, uint8_t* dest, int destSeek, int destSize);
4 changes: 4 additions & 0 deletions include/utils/LutLoader.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#ifndef PCH_ENABLED
#include <QString>
#include <QList>
#include <QFile>
#endif

#include <utils/PixelFormat.h>
Expand All @@ -17,4 +18,7 @@ class LutLoader {
MemoryBuffer<uint8_t> _lut;

void loadLutFile(Logger* _log, PixelFormat color, const QList<QString>& files);
private:
bool decompressLut(Logger* _log, QFile& file, int index);
void hasher(int index, Logger* _log);
};
97 changes: 97 additions & 0 deletions sources/utils-xz/utils-xz.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
#include <fstream>
#include <vector>
#include <lzma.h>
#include <cstring>
#include <utils-xz/utils-xz.h>

_XZ_SHARED_API const char* DecompressXZ(size_t downloadedDataSize, const uint8_t* downloadedData, const char* fileName)
Expand Down Expand Up @@ -83,3 +84,99 @@ _XZ_SHARED_API const char* DecompressXZ(size_t downloadedDataSize, const uint8_t

return error;
}

_XZ_SHARED_API const char* DecompressXZ(size_t downloadedDataSize, const uint8_t* downloadedData, uint8_t* dest, int destSeek, int destSize)
{
size_t outSize = 16842808/2;
std::vector<uint8_t> outBuffer;
const char* error = nullptr;
int currentPos = 0;
int writePos = 0;

try
{
outBuffer.resize(outSize);
}
catch (...)
{
error = "Could not allocate buffer";
}

if (error == nullptr)
{
const uint32_t flags = LZMA_TELL_UNSUPPORTED_CHECK | LZMA_CONCATENATED;
lzma_stream strm = LZMA_STREAM_INIT;
strm.next_in = downloadedData;
strm.avail_in = downloadedDataSize;
lzma_ret lzmaRet = lzma_stream_decoder(&strm, outSize, flags);
if (lzmaRet == LZMA_OK)
{
do {
strm.next_out = outBuffer.data();
strm.avail_out = outSize;
lzmaRet = lzma_code(&strm, LZMA_FINISH);
if (lzmaRet == LZMA_MEMLIMIT_ERROR)
{
outSize = lzma_memusage(&strm);
try
{
outBuffer.resize(outSize);
}
catch (...)
{
error = "Could not increase buffer size";
break;
}
lzma_memlimit_set(&strm, outSize);
strm.avail_out = 0;
}
else if (lzmaRet != LZMA_OK && lzmaRet != LZMA_STREAM_END)
{
error = "LZMA decoder returned error";
break;
}
else
{
int toWrite = outSize - strm.avail_out;
int endPos = currentPos + toWrite;
int chunkSize = -1, chunkPos = 0;

if (currentPos <= destSeek && destSeek + destSize >= endPos)
{
chunkPos = destSeek - currentPos;
chunkSize = toWrite - chunkPos;
}
else if (currentPos <= destSeek + destSize && destSeek + destSize < endPos)
{
chunkSize = (destSeek + destSize) - currentPos;
}
else if (currentPos >= destSeek && destSeek + destSize >= endPos)
{
chunkSize = toWrite;
}

if (chunkSize > 0)
{
memcpy(dest + writePos, outBuffer.data() + chunkPos, chunkSize);
writePos += chunkSize;
}

currentPos = endPos;
}
} while (strm.avail_out == 0 && lzmaRet != LZMA_STREAM_END && (writePos < destSize));
}
else
{
error = "Could not initialize LZMA decoder";
}

lzma_end(&strm);
}

if (writePos != destSize)
{
error = "Incorrect final LUT size";
}

return error;
}
92 changes: 80 additions & 12 deletions sources/utils/LutLoader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,18 @@
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/

#include <HyperhdrConfig.h>
#include <utils/LutLoader.h>
#include <image/ColorRgb.h>
#include <utils/FrameDecoder.h>
#include <utils/InternalClock.h>
#include <QFile>

#ifdef ENABLE_XZ
#include <utils-xz/utils-xz.h>
#endif


namespace {
const int LUT_FILE_SIZE = 256 * 256 * 256 *3;
const int LUT_MEMORY_ALIGN = 64;
Expand All @@ -52,17 +58,26 @@ void LutLoader::loadLutFile(Logger* _log, PixelFormat color, const QList<QString
for (QString fileName3d : files)
{
QFile file(fileName3d);
bool compressed = false;

if (file.open(QIODevice::ReadOnly))
if (file.open(QIODevice::ReadOnly)
#ifdef ENABLE_XZ
|| [&]() {file.setFileName(fileName3d + ".xz"); compressed = true; return file.open(QIODevice::ReadOnly); } ()
#endif
)
{
int length;
if (_log) Debug(_log, "LUT file found: %s", QSTRING_CSTR(fileName3d));

if (_log)
{
Debug(_log, "LUT file found: %s (%s)", QSTRING_CSTR(file.fileName()), (compressed) ? "compressed" : "uncompressed");
}

length = file.size();

if ((length == LUT_FILE_SIZE * 3) || (length == LUT_FILE_SIZE && !is_yuv))
if ((length == LUT_FILE_SIZE * 3) || (length == LUT_FILE_SIZE && !is_yuv) || compressed)
{
qint64 index = 0;
int index = 0;

if (is_yuv && _hdrToneMappingEnabled)
{
Expand All @@ -77,21 +92,35 @@ void LutLoader::loadLutFile(Logger* _log, PixelFormat color, const QList<QString
else
{
if (_log) Debug(_log, "Index 0 for HDR RGB");
}

file.seek(index);
}

_lut.resize(LUT_FILE_SIZE + LUT_MEMORY_ALIGN);

if (file.read((char*)_lut.data(), LUT_FILE_SIZE) != LUT_FILE_SIZE)
if (!compressed)
{
if (_log) Error(_log, "Error reading LUT file %s", QSTRING_CSTR(fileName3d));
file.seek(index);

if (file.read((char*)_lut.data(), LUT_FILE_SIZE) != LUT_FILE_SIZE)
{
if (_log) Error(_log, "Error reading LUT file %s", QSTRING_CSTR(fileName3d));
}
else
{
_lutBufferInit = true;
if (_log) Info(_log, "Found and loaded LUT: '%s'", QSTRING_CSTR(fileName3d));
}
}
else
{
_lutBufferInit = true;
if (_log) Info(_log, "Found and loaded LUT: '%s'", QSTRING_CSTR(fileName3d));
_lutBufferInit = decompressLut(_log, file, index);
if (_log)
{
if (_lutBufferInit) Info(_log, "Found and loaded LUT: '%s'", QSTRING_CSTR(fileName3d));
else Error(_log, "Error reading LUT file %s", QSTRING_CSTR(fileName3d));
}
}

// hasher(index / LUT_FILE_SIZE, _log);
}
else
{
Expand All @@ -111,3 +140,42 @@ void LutLoader::loadLutFile(Logger* _log, PixelFormat color, const QList<QString
if (_log) Error(_log, "Could not find any required LUT file");
}
}

bool LutLoader::decompressLut(Logger* _log, QFile& file, int index)
{
auto now = InternalClock::nowPrecise();
const char* retVal = "HyperHDR was built without a support for XZ decoder";
QByteArray compressedFile = file.readAll();
#ifdef ENABLE_XZ
retVal = DecompressXZ(compressedFile.size(), reinterpret_cast<uint8_t*>(compressedFile.data()), _lut.data(), index, LUT_FILE_SIZE);
#endif

if (retVal != nullptr && _log)
{
Error(_log, "Error while decompressing LUT: %s", retVal);
}

if (_log) Info(_log, "Decompression took %f seconds", (InternalClock::nowPrecise() - now) / 1000.0);

return retVal == nullptr;
}

void LutLoader::hasher(int index, Logger* _log)
{
if (_log)
{
auto start = _lut.data();
auto end = start + _lut.size();
uint8_t position = 0;
uint16_t fletcher1 = 0;
uint16_t fletcher2 = 0;
uint16_t fletcherExt = 0;
while (start < end)
{
fletcher1 = (fletcher1 + (uint16_t)(*(start))) % 255;
fletcher2 = (fletcher2 + fletcher1) % 255;
fletcherExt = (fletcherExt + (*(start++) ^ (position++))) % 255;
}
Info(_log, "CRC for %i segment: 0x%.2X 0x%.2X 0x%.2X", index, fletcher1, fletcher2, fletcherExt);
}
}

0 comments on commit a47d436

Please sign in to comment.