diff --git a/Project/GNU/CLI/test/paddingbits.txt b/Project/GNU/CLI/test/paddingbits.txt index 16e29775..c253145a 100644 --- a/Project/GNU/CLI/test/paddingbits.txt +++ b/Project/GNU/CLI/test/paddingbits.txt @@ -26,14 +26,23 @@ Features/AV_Package/818_DCDM_P3_IA_FIC_000918/818_OV 1 pass --all Features/AV_Package/818_DCDM_P3_IA_FIC_000918/818_OV 1 fail --check --no-check-padding Features/AV_Package/818_DCDM_P3_IA_FIC_000918/818_OV 1 pass --check --no-check-padding --quick-check-padding Features/HashCheck/HashTest_BuggyFileHash.mkv X pass --check --no-hash -Features/HashCheck/HashTest_BuggyHashInternal.mkv X pass --check --no-hash -Features/HashCheck/HashTest_BuggyHashInternal_BuggyFileHash.mkv X pass --check --no-hash -Features/HashCheck/HashTest_BuggyHashInternal_FileHash.mkv X pass --check --no-hash +Features/HashCheck/HashTest_BuggyHashInternal.mkv X fail --check --no-hash +Features/HashCheck/HashTest_BuggyHashInternal_BuggyFileHash.mkv X fail --check --no-hash +Features/HashCheck/HashTest_BuggyHashInternal_FileHash.mkv X fail --check --no-hash Features/HashCheck/HashTest_FileHash.mkv X pass --check --no-hash Features/HashCheck/HashTest_HashInternal.mkv X pass --check --no-hash Features/HashCheck/HashTest_HashInternal_BuggyFileHash.mkv X pass --check --no-hash Features/HashCheck/HashTest_HashInternal_FileHash.mkv X pass --check --no-hash Features/HashCheck/HashTest_NoHash.mkv X pass --check --no-hash +Features/HashCheck/HashTest_BuggyFileHash.mkv X pass --check --info +Features/HashCheck/HashTest_BuggyHashInternal.mkv X fail --check --info +Features/HashCheck/HashTest_BuggyHashInternal_BuggyFileHash.mkv X fail --check --info +Features/HashCheck/HashTest_BuggyHashInternal_FileHash.mkv X fail --check --info +Features/HashCheck/HashTest_FileHash.mkv X pass --check --info +Features/HashCheck/HashTest_HashInternal.mkv X pass --check --info +Features/HashCheck/HashTest_HashInternal_BuggyFileHash.mkv X pass --check --info +Features/HashCheck/HashTest_HashInternal_FileHash.mkv X pass --check --info +Features/HashCheck/HashTest_NoHash.mkv X pass --check --info Features/HashCheck/HashTest_BuggyFileHash.mkv X pass --check Features/HashCheck/HashTest_BuggyHashInternal.mkv X fail --check Features/HashCheck/HashTest_BuggyHashInternal_BuggyFileHash.mkv X fail --check diff --git a/Source/CLI/Global.cpp b/Source/CLI/Global.cpp index 72325c42..2f875984 100644 --- a/Source/CLI/Global.cpp +++ b/Source/CLI/Global.cpp @@ -68,6 +68,8 @@ int global::SetCheck(bool Value) { Actions.set(Action_Check, Value); Actions.set(Action_CheckOptionIsSet); + if (Value) + return SetDecode(false); return 0; } @@ -152,6 +154,15 @@ int global::SetEncode(bool Value) return 0; } +//--------------------------------------------------------------------------- +int global::SetInfo(bool Value) +{ + Actions.set(Action_Info, Value); + if (Value) + return SetDecode(false); + return 0; +} + //--------------------------------------------------------------------------- int global::SetFrameMd5(bool Value) { @@ -170,13 +181,14 @@ int global::SetFrameMd5FileName(const char* FileName) int global::SetHash(bool Value) { Actions.set(Action_Hash, Value); - Actions.set(Action_HashOptionIsSet); return 0; } //--------------------------------------------------------------------------- int global::SetAll(bool Value) { + if (int ReturnValue = SetInfo(Value)) + return ReturnValue; if (int ReturnValue = (Value?SetCheck(true):SetQuickCheck())) // Never implicitely set no check return ReturnValue; if (int ReturnValue = (Value && SetCheckPadding(Value))) // Never implicitely set no check padding @@ -543,6 +555,12 @@ int global::ManageCommandLine(const char* argv[], int argc) if (Value) return Value; } + else if (strcmp(argv[i], "--info") == 0) + { + int Value = SetInfo(true); + if (Value) + return Value; + } else if ((strcmp(argv[i], "--license") == 0 || strcmp(argv[i], "--licence") == 0)) { if (i + 1 == argc) @@ -593,6 +611,12 @@ int global::ManageCommandLine(const char* argv[], int argc) if (Value) return Value; } + else if (strcmp(argv[i], "--no-info") == 0) + { + int Value = SetInfo(false); + if (Value) + return Value; + } else if (strcmp(argv[i], "--none") == 0) { int Value = SetAll(false); diff --git a/Source/CLI/Global.h b/Source/CLI/Global.h index 2d735014..3b5d7b24 100644 --- a/Source/CLI/Global.h +++ b/Source/CLI/Global.h @@ -90,6 +90,7 @@ class global int SetConch(bool Value); int SetDecode(bool Value); int SetEncode(bool Value); + int SetInfo(bool Value); int SetFrameMd5(bool Value); int SetFrameMd5FileName(const char* FileName); int SetHash(bool Value); diff --git a/Source/CLI/Help.cpp b/Source/CLI/Help.cpp index 3f899c24..6e847d58 100644 --- a/Source/CLI/Help.cpp +++ b/Source/CLI/Help.cpp @@ -110,18 +110,19 @@ ReturnValue Help(const char* Name) cout << endl; cout << "ACTIONS" << endl; cout << " --all" << endl; - cout << " Same as --decode --encode --hash --conch --coherency" << endl; - cout << " --check-padding --check"<< endl; + cout << " Same as --info --decode --encode --hash --conch" << endl; + cout << " --coherency --check-padding --check"<< endl; cout << " --none" << endl; - cout << " Same as --no-decode --no-encode --no-hash --no-conch" << endl; - cout << " --no-coherency --quick-check" << endl; + cout << " Same as --no-info --no-decode --no-encode --no-hash" << endl; + cout << " --no-conch --no-coherency --quick-check-padding --quick-check" << endl; cout << endl; cout << " --check" << endl; cout << " Check that the encoded file can be correctly decoded." << endl; - cout << " If input is raw content, encode then decode input and check" << endl; - cout << " that output is same as the input content." << endl; - cout << " If input is compressed content, decode input and check" << endl; - cout << " that output is same as the decoded content." << endl; + cout << " If input is raw content, encode then check that output" << endl; + cout << " would be same as the input content." << endl; + cout << " If input is compressed content, check that output" << endl; + cout << " would be same as the original content." << endl; + cout << " Disable decoding." << endl; cout << " --quick-check" << endl; cout << " Do quick coherency checks of the encoded file." << endl; cout << " It permits to check that the file seems healthy" << endl; @@ -131,6 +132,14 @@ ReturnValue Help(const char* Name) cout << " --no-check" << endl; cout << " Don't do checks (see above)." << endl; cout << endl; + cout << " --info" << endl; + cout << " Provide some extra information about the compresssed file," << endl; + cout << " for example the presence of hash of the raw data." << endl; + cout << " Disable decoding." << endl; + cout << " --no-info" << endl; + cout << " Don't provide some extra information (see above)." << endl; + cout << " Is default (it may change in the future)" << endl; + cout << endl; cout << " --check-padding" << endl; cout << " Run padding checks. Be aware check function can be demanding" << endl; cout << " of time and processor usage." << endl; @@ -176,23 +185,13 @@ ReturnValue Help(const char* Name) cout << " Don't encode (see above)." << endl; cout << endl; cout << " --hash" << endl; - cout << " Compute or test the hash of files." << endl; + cout << " Compute the hash of audio-visual RAW data files." << endl; cout << " If input is raw content, do the hash and store hashes in RAWcooked" << endl; cout << " metadata." << endl; - cout << " If an hash file is in the raw content, test that the hash file is" << endl; - cout << " valid." << endl; - cout << " If input is a Matroska container and RAWcooked metadata contains" << endl; - cout << " hashes decode input and check hashes (issue considered as decoding" << endl; - cout << " error)." << endl; - cout << " If input is a Matroska container and with hash files in" << endl; - cout << " attachment, decode input and check hashes (issue considered as" << endl; - cout << " invalid data)." << endl; - cout << " Is default if input is a Matroska container and RAWcooked metadata" << endl; - cout << " contains hashes or if input is a Matroska container and with hash" << endl; - cout << " files in attachment (it may change in the future)" << endl; + cout << " This permits a reversibility check without the original files." << endl; cout << " --no-hash" << endl; cout << " Don't do compute or test of hash of files. (see above)." << endl; - cout << " Is default if input is raw content (it may change in the future)" << endl; + cout << " Is default (it may change in the future)" << endl; cout << endl; cout << " --framemd5" << endl; cout << " Compute the framemd5 of input frames and store it to a sidecar" << endl; diff --git a/Source/CLI/Main.cpp b/Source/CLI/Main.cpp index ef0b1964..b7d82e34 100644 --- a/Source/CLI/Main.cpp +++ b/Source/CLI/Main.cpp @@ -386,15 +386,37 @@ int ParseFile_Compressed(parse_info& ParseInfo) matroska* M = new matroska(OutputDirectoryName, &Global.Mode, Ask_Callback, Thread_Pool, &Global.Errors); M->Quiet = Global.Quiet; - M->NoWrite = Global.Actions[Action_Check] || !Global.Actions[Action_Decode]; M->NoOutputCheck = NoOutputCheck; - M->NoHashCheck = Global.Actions[Action_HashOptionIsSet] && !Global.Actions[Action_Hash]; if (ParseInfo.ParseFile_Input(*M)) { ReturnValue = 1; } - if (!HasCheckedReversibility && M->Hashes_FromRAWcooked) - HasCheckedReversibility = true; + else if (M->IsDetected()) + { + if (!HasCheckedReversibility && M->Hashes_FromRAWcooked) + HasCheckedReversibility = true; + + if (Global.Actions[Action_Info]) + { + if (!M->RAWcooked_LibraryNameVersion_Get().empty()) + { + cout << "\nInfo: Reversibility data created by " << M->RAWcooked_LibraryNameVersion_Get() << '.'; + } + else + { + cout << "\nInfo: No reversibility data found."; + } + if (M->Hashes_FromRAWcooked) + { + cout << "\nInfo: Uncompressed file hashes (used by reversibility check) present."; + } + if (M->Hashes_FromAttachments && M->Hashes_FromAttachments->HashFiles_Count()) + { + cout << "\nInfo: " << M->Hashes_FromAttachments->HashFiles_Count() << " hash file (used by conformance check) found."; + } + cout << endl; + } + } delete M; delete Thread_Pool; } @@ -402,9 +424,9 @@ int ParseFile_Compressed(parse_info& ParseInfo) // End if (ParseInfo.IsDetected && !Global.Quiet) { - if (!Global.Actions[Action_Check] && Global.Actions[Action_Decode]) + if (Global.Actions[Action_Decode]) cout << "\nFiles are in " << OutputDirectoryName << '.' << endl; - else if (Global.Actions[Action_Check] && !Global.Errors.HasErrors()) + if (Global.Actions[Action_Check] && !Global.Errors.HasErrors()) cout << '\n' << (HasCheckedReversibility ? "Reversability" : "Decoding") << " was checked, no issue detected." << endl; } if (Global.Actions[Action_Check] && Global.Errors.HasErrors()) @@ -550,7 +572,7 @@ int main(int argc, const char* argv[]) // Parse (check mode) Global.Actions.set(Action_QuickCheckAfterEncode, !Global.Actions[Action_Check]); - Global.Actions.set(Action_Decode, true); // Override config + Global.Actions.set(Action_Decode, false); // Override config Value = ParseFile_Compressed(ParseInfo); if (!Value && !ParseInfo.IsDetected) { diff --git a/Source/CLI/rawcooked.1 b/Source/CLI/rawcooked.1 index 25d9db49..3d3a0cde 100644 --- a/Source/CLI/rawcooked.1 +++ b/Source/CLI/rawcooked.1 @@ -92,17 +92,19 @@ Assume \fIno\fR as answer to all prompts, and run non-interactively. .SS ACTIONS .TP .B --all -Same as --decode --encode --hash --conch --coherency --check-padding --check (see below) +Same as --info --decode --encode --hash --conch --coherency --check-padding --check (see below) .TP .B --none -Same as --no-decode --no-encode --no-hash --no-conch --no-coherency --quick-check (see below) +Same as --no-info --no-decode --no-encode --no-hash --no-conch --no-coherency --quick-check-padding --quick-check (see below) .TP .B --check Check that the encoded file can be correctly decoded. .br -If the input is raw content such as DPX, encode then decode the raw input, before checking that the original raw content and the decoded files are identical. +If input is raw content, encode then check that output would be same as the input content. .br -If the input is a Matroska container, decode the input and check that the output is same as the decoded content. +If input is compressed content, check that output would be same as the original content. +.br +Disable decoding. .TP .B --quick-check Do quick coherency checks of the encoded file. It permits to check that the file seems healthy without the time and processor usage of the full check. @@ -116,6 +118,16 @@ Don't run any checks (see above). .br This is the default, but may change in the future. .TP +.B --info +Provide extra information about the compresssed file, for example the presence of hash of the raw data. +.br +Disable decoding. +.TP +.B --no-info +Don't provide extra information (see above). +.br +This is the default, but may change in the future. +.TP .B --check-padding Run padding checks. Be aware check function can be demanding of time and processor usage. .br @@ -170,22 +182,14 @@ This is default. Do not carry out encode (see above). .TP .B --hash -Can be used to either compute, or test, the hash of files. +Compute the hash of audio-visual RAW data files. .br -To compute a hash use with the audio-visual RAW data. A hash is generated and stored in the RAWcooked metadata. -.br -Repeat the command with the RAW data to test that the previously generated hash file is valid. -.br -If the input is a Matroska container with metadata that contains computed hashes then decode the Matroska input and check hashes. Any issue raised by the check is considered a decoding error. -.br -If the input is a Matroska container with computed hash files stored in an attachment, then decode the Matroska input and check hashes. Any issue raised by the check is considered as invalid data. -.br -This is default when the input is a Matroska container and RAWcooked metadata contains hashes, and when the input is a Matroska container with hash files in the attachment. This default may change in the future. +This permits a reversibility check without the original files. .TP .B --no-hash Do not compute or test the hash of the file (see above). .br -This is default when the input is RAW content, but may change in the future. +This is default, but may change in the future. .TP .B --framemd5 Compute the framemd5 of input frames and store it to a sidecar file. diff --git a/Source/Lib/Compressed/Matroska/Matroska.cpp b/Source/Lib/Compressed/Matroska/Matroska.cpp index 239e7b7f..507b9a2e 100644 --- a/Source/Lib/Compressed/Matroska/Matroska.cpp +++ b/Source/Lib/Compressed/Matroska/Matroska.cpp @@ -211,13 +211,13 @@ void matroska::Shutdown() TrackInfo.clear(); // Hashes - if (Hashes_FromRAWcooked) + if ((Actions[Action_Decode] || Actions[Action_Check] || Actions[Action_Conch]) && Hashes_FromRAWcooked) { - if (ReversibilityCompat >= Compat_18_10_1) + if (ReversibilityCompat > Compat_18_10_1) Hashes_FromRAWcooked->RemoveEmptyFiles(); Hashes_FromRAWcooked->Finish(); } - if (Hashes_FromAttachments) + if (Actions[Action_Conch] && Hashes_FromAttachments) { Hashes_FromAttachments->RemoveEmptyFiles(); // Attachments don't have files with a size of 0 Hashes_FromAttachments->Finish(); @@ -246,7 +246,6 @@ void matroska::Shutdown() } ReversibilityFileHasNames = true; - break; } } @@ -306,15 +305,13 @@ void matroska::ParseBuffer() ProgressIndicator_Thread=new thread(matroska_ProgressIndicator_Show, this); // Config - if (NoWrite) + if (!Actions[Action_Decode]) FrameWriter_Template->Mode.set(frame_writer::NoWrite); if (NoOutputCheck) FrameWriter_Template->Mode.set(frame_writer::NoOutputCheck); - if (NoHashCheck) - FrameWriter_Template->Mode.set(frame_writer::NoHashCheck); - else + if (Actions[Action_Decode] || Actions[Action_Check] || Actions[Action_Conch] || Actions[Action_Info]) Hashes_FromRAWcooked = new hashes(Errors); - if (Actions[Action_Conch]) + if (Actions[Action_Conch] || Actions[Action_Info]) Hashes_FromAttachments = new hashes(Errors); Levels[Level].Offset_End = Buffer.Size(); @@ -387,7 +384,7 @@ void matroska::Segment() IsList = true; // In case of partial check - if (Actions[Action_QuickCheckAfterEncode]) // Quick check after encoding + if (!Actions[Action_Decode] && !Actions[Action_Info] && Actions[Action_QuickCheckAfterEncode]) // Quick check after encoding { Buffer_Offset = Buffer.Size(); Shutdown(); @@ -448,8 +445,6 @@ void matroska::Segment_Attachments_AttachedFile_FileData() HashSum.Parse(buffer_view(Buffer.Data() + Buffer_Offset, Levels[Level].Offset_End - Buffer_Offset)); if (HashSum.IsDetected()) { - if (Hashes_FromRAWcooked) - Hashes_FromRAWcooked->Ignore(AttachedFile_FileName); if (Hashes_FromAttachments) Hashes_FromAttachments->Ignore(AttachedFile_FileName); AttachedFile_FileNames_IsHash.insert(AttachedFile_FileName); @@ -457,6 +452,7 @@ void matroska::Segment_Attachments_AttachedFile_FileData() } // Output file + if (Actions[Action_Decode] || Actions[Action_Check] || Actions[Action_Conch]) { raw_frame RawFrame; RawFrame.SetPre(buffer_view(Buffer.Data() + Buffer_Offset, Levels[Level].Offset_End - Buffer_Offset)); @@ -663,14 +659,23 @@ void matroska::Segment_Cluster() // Check if Hashes check is useful if (Hashes_FromRAWcooked) { + for (const auto& AttachedFile : AttachedFiles) + { + if (ReversibilityCompat >= Compat_18_10_1 && !AttachedFile_FileNames_IsHash.empty()) // In previous versions hash files were not listed in reversibility file + { + for (const auto& Name : AttachedFile_FileNames_IsHash) + Hashes_FromRAWcooked->Ignore(Name); + } + } + Hashes_FromRAWcooked->WouldBeError = true; - if (!Hashes_FromRAWcooked->NoMoreHashFiles()) + if (!(Actions[Action_Decode] || Actions[Action_Check] || Actions[Action_Conch]) || !Hashes_FromRAWcooked->NoMoreHashFiles()) { delete Hashes_FromRAWcooked; Hashes_FromRAWcooked = nullptr; } } - if (Hashes_FromAttachments) + if (Actions[Action_Conch] && Hashes_FromAttachments) { if (!Hashes_FromAttachments->NoMoreHashFiles()) { @@ -679,6 +684,12 @@ void matroska::Segment_Cluster() } } + if (!Actions[Action_Decode] && !Actions[Action_Check] && !Actions[Action_Conch]) // No file parsing requested, we stop now + { + Buffer_Offset = Buffer.Size(); + return; + } + // Init for (const auto& TrackInfo_Current : TrackInfo) if (TrackInfo_Current && TrackInfo_Current->Init(Buffer.Data())) diff --git a/Source/Lib/Compressed/Matroska/Matroska.h b/Source/Lib/Compressed/Matroska/Matroska.h index b1f37e03..737b7e90 100644 --- a/Source/Lib/Compressed/Matroska/Matroska.h +++ b/Source/Lib/Compressed/Matroska/Matroska.h @@ -47,9 +47,7 @@ class matroska : public input_base void Shutdown(); bool Quiet = false; - bool NoWrite = false; bool NoOutputCheck = false; - bool NoHashCheck = false; hashes* Hashes_FromRAWcooked = nullptr; hashes* Hashes_FromAttachments = nullptr; @@ -67,6 +65,9 @@ class matroska : public input_base Track_MaskBase, }; + // Info + string RAWcooked_LibraryNameVersion_Get() { if (RAWcooked_LibraryName.empty()) return string(); return RAWcooked_LibraryName + ' ' + RAWcooked_LibraryVersion; } + private: void ParseBuffer(); void BufferOverflow(); diff --git a/Source/Lib/Compressed/RAWcooked/Track.cpp b/Source/Lib/Compressed/RAWcooked/Track.cpp index 08797bd4..9084878d 100644 --- a/Source/Lib/Compressed/RAWcooked/Track.cpp +++ b/Source/Lib/Compressed/RAWcooked/Track.cpp @@ -212,6 +212,11 @@ input_base_uncompressed* track_info::InitOutput(input_base_uncompressed* Potenti //--------------------------------------------------------------------------- void track_info::End(size_t i) { + if (!Actions[Action_Decode] && !Actions[Action_Check]) + { + return; + } + // Write end of the file if the file is unique per track if (ReversibilityData->Unique()) { diff --git a/Source/Lib/Uncompressed/HashSum/HashSum.cpp b/Source/Lib/Uncompressed/HashSum/HashSum.cpp index f6564ae3..c26a4f45 100644 --- a/Source/Lib/Uncompressed/HashSum/HashSum.cpp +++ b/Source/Lib/Uncompressed/HashSum/HashSum.cpp @@ -189,7 +189,7 @@ void hashes::Finish() if (!HashItem.Flags[hashes::value::Flag_IsFound] && find(HashFiles.begin(), HashFiles.end(), HashItem.Name) == HashFiles.end()) { if (Errors) - Errors->Error(IO_Hashes, CheckFromFiles ? error::type::Undecodable : error::type::Invalid, CheckFromFiles ? (error::generic::code)hashes_issue::undecodable::FileExtra : (error::generic::code)hashes_issue::invalid::FileHashExtra, HashItem.Name); + Errors->Error(IO_Hashes, WouldBeError ? error::type::Undecodable : error::type::Invalid, CheckFromFiles ? (error::generic::code)hashes_issue::undecodable::FileExtra : (error::generic::code)hashes_issue::invalid::FileHashExtra, HashItem.Name); } } } diff --git a/Source/Lib/Uncompressed/HashSum/HashSum.h b/Source/Lib/Uncompressed/HashSum/HashSum.h index 89e7732a..7f5197e0 100644 --- a/Source/Lib/Uncompressed/HashSum/HashSum.h +++ b/Source/Lib/Uncompressed/HashSum/HashSum.h @@ -62,6 +62,9 @@ class hashes void FromFile(string const& FileName, md5 const& MD5); void Finish(); + // Info + size_t HashFiles_Count() { return HashFiles.size(); } + private: // Internal void NoMoreHashFiles_Internal(); diff --git a/Source/Lib/Utils/FileIO/FileWriter.h b/Source/Lib/Utils/FileIO/FileWriter.h index 9a7ecee7..0e47c859 100644 --- a/Source/Lib/Utils/FileIO/FileWriter.h +++ b/Source/Lib/Utils/FileIO/FileWriter.h @@ -57,7 +57,6 @@ class frame_writer : public raw_frame_process IsNotEnd, NoWrite, NoOutputCheck, - NoHashCheck, mode_Max, }; bitset Mode; diff --git a/Source/Lib/Utils/FileIO/Input_Base.h b/Source/Lib/Utils/FileIO/Input_Base.h index 0214125e..faf76b97 100644 --- a/Source/Lib/Utils/FileIO/Input_Base.h +++ b/Source/Lib/Utils/FileIO/Input_Base.h @@ -26,7 +26,6 @@ enum action : uint8_t Action_Encode, Action_Decode, Action_Hash, - Action_HashOptionIsSet, Action_Coherency, Action_Conch, Action_CheckPadding, @@ -35,6 +34,7 @@ enum action : uint8_t Action_AcceptTruncated, Action_Check, Action_CheckOptionIsSet, + Action_Info, Action_FrameMd5, Action_QuickCheckAfterEncode, // Internal, indicating the 2nd pass Action_Max