diff --git a/.clang-format b/.clang-format index e93db93..dcca22e 100644 --- a/.clang-format +++ b/.clang-format @@ -6,6 +6,7 @@ PointerAlignment: Left ExperimentalAutoDetectBinPacking: true SpaceBeforeCpp11BracedList: true IndentPPDirectives: BeforeHash +IncludeBlocks: Regroup --- Language: ObjC # Force pointers to the type for C++. diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f978387..2b6be01 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,8 +18,8 @@ jobs: run: make - name: Test binary run: | - ./aastuff - ./aastuff_standalone + ./aastuff -h + ./aastuff_standalone -h - name: Upload artifacts uses: actions/upload-artifact@v4 with: @@ -60,6 +60,7 @@ jobs: unzip -p tests/iPhone_15PM_18.0_22A5282m.ipsw 090-29713-049.dmg.aea > tests/iOS_18_beta_1_IPSW/encrypted.aea rm tests/iPhone_15PM_18.0_22A5282m.ipsw printf "$IOS_IPSW_TEST_KEY" > tests/iOS_18_beta_1_IPSW/expected.txt + touch tests/iOS_18_beta_1_IPSW/fast_unsupported env: MACOS_OTA_TEST_KEY: ${{ vars.MACOS_OTA_TEST_KEY }} IOS_IPSW_TEST_KEY: ${{ vars.IOS_IPSW_TEST_KEY }} @@ -86,11 +87,13 @@ jobs: # This file uses a compressed inner layer # curl -L "https://updates.cdn-apple.com/2024/Iris/mobileassets/003-49672/A1233F60-3D17-491B-803A-DB26E20695AE/com_apple_MobileAsset_UAF_Siri_Understanding/6FF3BAF0-FBEF-4C01-BB0E-30CD61DAFCC4.aar" -o tests/small/encrypted.aea aria2c -x 16 -s 16 -j 16 --file-allocation=none "https://updates.cdn-apple.com/2024/Iris/mobileassets/003-49672/A1233F60-3D17-491B-803A-DB26E20695AE/com_apple_MobileAsset_UAF_Siri_Understanding/6FF3BAF0-FBEF-4C01-BB0E-30CD61DAFCC4.aar" -o tests/small/encrypted.aea - printf "$SMALL_TEST_KEY" > tests/small/key.txt + printf "$SMALL_TEST_KEY" > tests/small/expected.txt + printf "YEC,UID,GID,MOD,SH2" > tests/small/flags.txt # This file uses a raw inner layer # curl -L "https://updates.cdn-apple.com/2024SummerSeed/mobileassets/052-49061/CA7135A8-BAF6-4890-887C-35FB30C154D5/com_apple_MobileAsset_MacSoftwareUpdate/e2de87f20576b2bdc021d36f74a2f836cf42afe576178388dfd0cde875f4f979.aea" -o tests/large/encrypted.aea aria2c -x 16 -s 16 -j 16 --file-allocation=none "https://updates.cdn-apple.com/2024SummerSeed/mobileassets/052-49061/CA7135A8-BAF6-4890-887C-35FB30C154D5/com_apple_MobileAsset_MacSoftwareUpdate/e2de87f20576b2bdc021d36f74a2f836cf42afe576178388dfd0cde875f4f979.aea" -o tests/large/encrypted.aea - printf "$LARGE_TEST_KEY" > tests/large/key.txt + printf "$LARGE_TEST_KEY" > tests/large/expected.txt + printf "LNK,FLG,UID,GID,MOD,MTM,CTM" > tests/large/flags.txt env: SMALL_TEST_KEY: ${{ vars.SMALL_TEST_KEY }} LARGE_TEST_KEY: ${{ vars.LARGE_TEST_KEY }} diff --git a/Makefile b/Makefile index 41ae4c1..429c042 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,9 @@ .PHONY: all clean test deploy -SRC_FILES = src/aastuff.m src/extract.m src/extract_standalone.m +SRC_FILES = src/aastuff.m src/args.m src/extract.m src/extract_standalone.m HDR_FILES = include/AppleArchivePrivate.h include/extract.h include/extract_standalone.h -CFLAGS = -fmodules -fobjc-arc -Iinclude -Wall -Werror +CFLAGS = -fmodules -fobjc-arc -Iinclude -Wall -Werror -Wunreachable-code LDLIBS = -framework Foundation -lAppleArchive LDFLAGS = -Llib diff --git a/include/AppleArchivePrivate.h b/include/AppleArchivePrivate.h index 02ae92e..19fd2be 100644 --- a/include/AppleArchivePrivate.h +++ b/include/AppleArchivePrivate.h @@ -9,11 +9,23 @@ __BEGIN_DECLS // TODO: Figure out how this is different from normal AppleArchive typedef void* AAAssetExtractor; -AAAssetExtractor AAAssetExtractorCreate(const char* destDir, void** something, int something2); -// AAAssetExtractorSetParameterCallback -// AAAssetExtractorSetParameterPtr +AAAssetExtractor AAAssetExtractorCreate(const char* work_dir, uint64_t* maybe_offset, void* unknown); + +// 102: AEA context +// 103: progress +// 104: entry message +// - 90: start? +// - 91: extract? +// - 92: complete +int AAAssetExtractorSetParameterCallback(AAAssetExtractor extractor, int param, void* callback); + +// 101: first arg to callbacks +// 105: input directory +// 106: output directory +int AAAssetExtractorSetParameterPtr(AAAssetExtractor extractor, int param, void* ptr); + int AAAssetExtractorWrite(AAAssetExtractor extractor, void* buffer, size_t size); -void AAAssetExtractorDestroy(AAAssetExtractor extractor); +int AAAssetExtractorDestroy(AAAssetExtractor extractor, uint64_t* maybe_offset); AAArchiveStream AAVerifyDirectoryArchiveOutputStreamOpen(const char* dir, AAFieldKeySet key_set, void* msg_data, AAEntryMessageProc msg_proc, AAFlagSet flags, int n_threads); @@ -23,7 +35,7 @@ typedef uint32_t AAYopType; APPLE_ARCHIVE_ENUM(AAYopTypes, uint32_t) { AA_YOP_TYPE_COPY = 'C', ///< copy AA_YOP_TYPE_EXTRACT = 'E', ///< extract - AA_YOP_TYPE_SRC_CHECK = 'I', ///< extract + AA_YOP_TYPE_SRC_CHECK = 'I', ///< source check AA_YOP_TYPE_MANIFEST = 'M', ///< manifest AA_YOP_TYPE_DST_FIXUP = 'O', ///< destination fixup AA_YOP_TYPE_PATCH = 'P', ///< patch diff --git a/include/args.h b/include/args.h new file mode 100644 index 0000000..73f2abb --- /dev/null +++ b/include/args.h @@ -0,0 +1,25 @@ +#ifndef ARGS_H +#define ARGS_H + +#import + +@interface ExtractionConfiguration : NSObject + +@property(nonatomic, assign) bool encrypted; +@property(nonatomic, assign) bool list; +@property(nonatomic, strong) NSString* archivePath; +@property(nonatomic, strong) NSString* outputDirectory; +@property(nonatomic, strong) NSData* key; +@property(nonatomic, strong) NSString* filter; +@property(nonatomic, strong) NSRegularExpression* regex; + +@property(nonatomic, strong) NSString* function; + +- (instancetype)copyWithFunction:(NSString*)function; + +@end + +ExtractionConfiguration* parseArgs(int argc, char** argv, int* returnCode); +int validateArgs(ExtractionConfiguration* config); + +#endif /* ARGS_H */ diff --git a/include/extract.h b/include/extract.h index e087757..f910881 100644 --- a/include/extract.h +++ b/include/extract.h @@ -1,9 +1,11 @@ #ifndef EXTRACT_H #define EXTRACT_H -#include -#include +#import +#import -int extractAsset(AAByteStream stream, NSString* outputDirectory); +#import "args.h" + +int extractAsset(AAByteStream stream, ExtractionConfiguration* config); #endif /* EXTRACT_H */ diff --git a/include/extract_standalone.h b/include/extract_standalone.h index c9d465a..265edc3 100644 --- a/include/extract_standalone.h +++ b/include/extract_standalone.h @@ -1,9 +1,11 @@ #ifndef EXTRACT_STANDALONE_H #define EXTRACT_STANDALONE_H -#include -#include +#import +#import -int extractAssetStandalone(AAByteStream stream, NSString* outputDirectory); +#import "args.h" + +int extractAssetStandalone(AAByteStream stream, ExtractionConfiguration* config); #endif /* EXTRACT_STANDALONE_H */ diff --git a/include/utils.h b/include/utils.h index 83d0667..9a2ca59 100644 --- a/include/utils.h +++ b/include/utils.h @@ -14,4 +14,12 @@ #define DBGLOG(x, ...) #endif +#if AASTUFF_STANDALONE + #define NAME @"aastuff_standalone" +#else + #define NAME @"aastuff" +#endif + +#define VERSION @"0.0.1" + #endif /* UTILS_H */ diff --git a/src/aastuff.m b/src/aastuff.m index e50dc50..74e48b9 100644 --- a/src/aastuff.m +++ b/src/aastuff.m @@ -1,87 +1,24 @@ #import #import +#import "args.h" #import "extract.h" #import "extract_standalone.h" #import "utils.h" -#define APPLE_ARCHIVE_MAGIC @"AA01" -#define APPLE_ENCRYPTED_ARCHIVE_MAGIC @"AEA1" - int main(int argc, char** argv) { @autoreleasepool { - NSError* error = nil; - - if (argc < 3) { - ERRLOG(@"Usage: %s [key in base64]", argv[0]); - ERRLOG(@"Key is required for encrypted archives"); - return argc == 1 ? 0 : 1; - } - - NSString* archivePath = [NSString stringWithUTF8String:argv[1]]; - NSString* outputDirectory = [NSString stringWithUTF8String:argv[2]]; - NSString* keyBase64 = nil; - if (argc > 3) { - keyBase64 = [NSString stringWithUTF8String:argv[3]]; + int ret = 0; + ExtractionConfiguration* config = parseArgs(argc, argv, &ret); + if (!config) { + return ret; } - if (!archivePath || !outputDirectory) { - ERRLOG(@"Failed to parse arguments"); - return 1; - } - - NSFileManager* fileManager = [NSFileManager defaultManager]; - - if (![fileManager fileExistsAtPath:archivePath]) { - ERRLOG(@"Archive does not exist"); + if (validateArgs(config)) { return 1; } - BOOL isDirectory = false; - if (![fileManager fileExistsAtPath:outputDirectory isDirectory:&isDirectory]) { - if (![fileManager createDirectoryAtPath:outputDirectory withIntermediateDirectories:NO attributes:nil error:&error]) { - ERRLOG(@"Failed to create directory: %@", error); - return 1; - } - } else { - if (!isDirectory) { - ERRLOG(@"Output path is not a directory"); - return 1; - } - } - - NSFileHandle* handle = [NSFileHandle fileHandleForReadingAtPath:archivePath]; - if (!handle) { - ERRLOG(@"Failed to open archive file"); - return 1; - } - - NSData* magic = [handle readDataUpToLength:4 error:&error]; - // If this fails, can't do anything about it, so just ignore the error - [handle closeAndReturnError:nil]; - - if (!magic || magic.length != 4) { - ERRLOG(@"Failed to read magic: %@", error); - return 1; - } - - bool encrypted = false; - NSString* magicStr = [[NSString alloc] initWithData:magic encoding:NSUTF8StringEncoding]; - if ([magicStr isEqualToString:APPLE_ENCRYPTED_ARCHIVE_MAGIC]) { - encrypted = true; - } else if ([magicStr isEqualToString:APPLE_ARCHIVE_MAGIC]) { - encrypted = false; - } else { - ERRLOG(@"Unknown magic: %@", magicStr); - return 1; - } - - if (encrypted && !keyBase64) { - ERRLOG(@"Encrypted archive requires key"); - return 1; - } - - AAByteStream stream = AAFileStreamOpenWithPath(archivePath.UTF8String, O_RDONLY, 0644); + AAByteStream stream = AAFileStreamOpenWithPath(config.archivePath.UTF8String, O_RDONLY, 0644); if (!stream) { ERRLOG(@"Failed to open archive file stream"); return 1; @@ -89,14 +26,7 @@ int main(int argc, char** argv) { AAByteStream decryptionStream = NULL; AEAContext decryptionContext = NULL; - if (encrypted) { - NSData* key = [[NSData alloc] initWithBase64EncodedString:keyBase64 options:0]; - if (!key) { - ERRLOG(@"Failed to parse key"); - AAByteStreamClose(stream); - return 1; - } - + if (config.encrypted) { decryptionContext = AEAContextCreateWithEncryptedStream(stream); if (!decryptionContext) { ERRLOG(@"Failed to create encrypted stream context"); @@ -104,7 +34,7 @@ int main(int argc, char** argv) { return 1; } - int ret = AEAContextSetFieldBlob(decryptionContext, AEA_CONTEXT_FIELD_SYMMETRIC_KEY, 0, key.bytes, key.length); + int ret = AEAContextSetFieldBlob(decryptionContext, AEA_CONTEXT_FIELD_SYMMETRIC_KEY, 0, config.key.bytes, config.key.length); if (ret != 0) { ERRLOG(@"Failed to set key"); AEAContextDestroy(decryptionContext); @@ -114,7 +44,7 @@ int main(int argc, char** argv) { decryptionStream = AEADecryptionInputStreamOpen(stream, decryptionContext, 0, 0); if (!decryptionStream) { - ERRLOG(@"Failed to open decryption stream"); + ERRLOG(@"Failed to open decryption stream (invalid key?)"); AEAContextDestroy(decryptionContext); AAByteStreamClose(stream); return 1; @@ -122,9 +52,9 @@ int main(int argc, char** argv) { } #if AASTUFF_STANDALONE - if (extractAssetStandalone(encrypted ? decryptionStream : stream, outputDirectory)) { + if (extractAssetStandalone(config.encrypted ? decryptionStream : stream, config)) { #else - if (extractAsset(encrypted ? decryptionStream : stream, outputDirectory)) { + if (extractAsset(config.encrypted ? decryptionStream : stream, config)) { #endif ERRLOG(@"Extracting asset failed"); AEAContextDestroy(decryptionContext); diff --git a/src/args.m b/src/args.m new file mode 100644 index 0000000..6018c53 --- /dev/null +++ b/src/args.m @@ -0,0 +1,256 @@ +#import "args.h" + +#import +#import + +#import "utils.h" + +#define APPLE_ARCHIVE_MAGIC @"AA01" +#define APPLE_ENCRYPTED_ARCHIVE_MAGIC @"AEA1" + +/* +Options: +-l, --list + List the contents of the archive instead of extracting it +-i, --input + Input archive to extract +-o, --output + Output directory for extracted files +-k, --key + Key in base64 format for encrypted archives +-h, --help + Display this help message +-v, --version + Display the version number +-f, --filter + Filter files by pattern +-r, --regex + Filter files by regex pattern +*/ + +static void usage(void) { + ERRLOG(@"Usage: %@ [options]", NAME); + ERRLOG(@"Options:"); +#if AASTUFF_STANDALONE + ERRLOG(@" -l, --list"); + ERRLOG(@" List the contents of the archive instead of extracting it"); +#endif + ERRLOG(@" -i, --input "); + ERRLOG(@" Input archive to extract"); + ERRLOG(@" -o, --output "); + ERRLOG(@" Output directory for extracted files"); + ERRLOG(@" -k, --key "); + ERRLOG(@" Key in base64 format for encrypted archives"); + ERRLOG(@" -h, --help"); + ERRLOG(@" Display this help message"); + ERRLOG(@" -v, --version"); + ERRLOG(@" Display the version number"); +#if AASTUFF_STANDALONE + ERRLOG(@" -f, --filter "); + ERRLOG(@" Filter files by glob pattern"); + ERRLOG(@" -r, --regex "); + ERRLOG(@" Filter files by regex pattern"); +#endif +} + +ExtractionConfiguration* parseArgs(int argc, char** argv, int* returnCode) { + // clang-format off + static struct option long_options[] = { + #if AASTUFF_STANDALONE + {"list", no_argument, 0, 'l'}, + #endif + {"input", required_argument, 0, 'i'}, + {"output", required_argument, 0, 'o'}, + {"key", required_argument, 0, 'k'}, + {"help", no_argument, 0, 'h'}, + {"version", no_argument, 0, 'v'}, + #if AASTUFF_STANDALONE + {"filter", required_argument, 0, 'f'}, + {"regex", required_argument, 0, 'r'}, + #endif + {0, 0, 0, 0}, + }; + // clang-format on + + int option_index = 0; + int c; + + bool list = false; + NSString* archivePath = nil; + NSString* outputDirectory = nil; + NSString* keyBase64 = nil; + NSString* filter = nil; + NSString* regexString = nil; + + while ((c = getopt_long(argc, argv, "-li:o:k:hvf:r:", long_options, &option_index)) != -1) { + switch (c) { +#if AASTUFF_STANDALONE + case 'l': + list = true; + break; +#endif + case 'i': + archivePath = [NSString stringWithUTF8String:optarg]; + break; + case 'o': + outputDirectory = [NSString stringWithUTF8String:optarg]; + break; + case 'k': + keyBase64 = [NSString stringWithUTF8String:optarg]; + break; + case 'h': + usage(); + *returnCode = 0; + return nil; + case 'v': + LOG(@"%@ %@", NAME, VERSION); + *returnCode = 0; + return nil; +#if AASTUFF_STANDALONE + case 'f': + filter = [NSString stringWithUTF8String:optarg]; + break; + case 'r': + regexString = [NSString stringWithUTF8String:optarg]; + break; +#endif + default: + ERRLOG(@"Unknown option"); + usage(); + *returnCode = 1; + return nil; + } + } + + if (!archivePath) { + ERRLOG(@"Input archive is required"); + usage(); + *returnCode = 1; + return nil; + } + + if (!list && !outputDirectory) { + ERRLOG(@"Output directory is required"); + usage(); + *returnCode = 1; + return nil; + } + + if (filter && regexString) { + ERRLOG(@"Cannot use both filter and regex options"); + usage(); + *returnCode = 1; + return nil; + } + + NSData* key = nil; + if (keyBase64) { + key = [[NSData alloc] initWithBase64EncodedString:keyBase64 options:0]; + if (!key) { + ERRLOG(@"Failed to decode key from base64"); + *returnCode = 1; + return nil; + } + } + + NSRegularExpression* regex = nil; + if (regexString) { + NSError* error = nil; + regex = [NSRegularExpression regularExpressionWithPattern:regexString options:0 error:&error]; + if (error) { + ERRLOG(@"Failed to compile regex: %@", error); + *returnCode = 1; + return nil; + } + } + + ExtractionConfiguration* config = [[ExtractionConfiguration alloc] init]; + config.list = list; + config.archivePath = archivePath; + config.outputDirectory = outputDirectory; + config.key = key; + config.filter = filter; + config.regex = regex; + + return config; +} + +int validateArgs(ExtractionConfiguration* config) { + NSError* error = nil; + NSFileManager* fileManager = [NSFileManager defaultManager]; + + if (![fileManager fileExistsAtPath:config.archivePath]) { + ERRLOG(@"Archive does not exist"); + return 1; + } + + if (!config.list) { + BOOL isDirectory = false; + if (![fileManager fileExistsAtPath:config.outputDirectory isDirectory:&isDirectory]) { + if (![fileManager createDirectoryAtPath:config.outputDirectory withIntermediateDirectories:NO attributes:nil error:&error]) { + ERRLOG(@"Failed to create directory: %@", error); + return 1; + } + } else { + if (!isDirectory) { + ERRLOG(@"Output path is not a directory"); + return 1; + } + } + } else { + // We need an output directory for processing to work. However, we will not mutate it + config.outputDirectory = NSTemporaryDirectory(); + } + + NSFileHandle* handle = [NSFileHandle fileHandleForReadingAtPath:config.archivePath]; + if (!handle) { + ERRLOG(@"Failed to open archive file"); + return 1; + } + + NSData* magic = [handle readDataUpToLength:4 error:&error]; + // If this fails, can't do anything about it, so just ignore the error + [handle closeAndReturnError:nil]; + + if (!magic || magic.length != 4) { + ERRLOG(@"Failed to read magic: %@", error); + return 1; + } + + NSString* magicStr = [[NSString alloc] initWithData:magic encoding:NSUTF8StringEncoding]; + if ([magicStr isEqualToString:APPLE_ENCRYPTED_ARCHIVE_MAGIC]) { + config.encrypted = true; + } else if ([magicStr isEqualToString:APPLE_ARCHIVE_MAGIC]) { + config.encrypted = false; + } else { + ERRLOG(@"Unknown magic: %@", magicStr); + return 1; + } + + if (config.encrypted && !config.key) { + ERRLOG(@"Encrypted archive requires key"); + return 1; + } + + return 0; +} + +@implementation ExtractionConfiguration + +- (nonnull id)copyWithZone:(nullable NSZone*)zone { + ExtractionConfiguration* copy = [[ExtractionConfiguration alloc] init]; + copy.list = self.list; + copy.filter = self.filter; + copy.regex = self.regex; + + copy.function = self.function; + return copy; +} + +- (instancetype)copyWithFunction:(NSString*)function { + ExtractionConfiguration* copy = [self copy]; + copy.function = function; + return copy; +} + +@end diff --git a/src/extract.m b/src/extract.m index 0dcf27b..ce66903 100644 --- a/src/extract.m +++ b/src/extract.m @@ -1,12 +1,12 @@ #import "extract.h" + #import "AppleArchivePrivate.h" #import "utils.h" #define ALLOC_SIZE 0x100000uLL -int extractAsset(AAByteStream stream, NSString* outputDirectory) { - void* something = NULL; - AAAssetExtractor extractor = AAAssetExtractorCreate(outputDirectory.UTF8String, &something, 0LL); +int extractAsset(AAByteStream stream, ExtractionConfiguration* config) { + AAAssetExtractor extractor = AAAssetExtractorCreate(config.outputDirectory.UTF8String, NULL, NULL); if (!extractor) { ERRLOG(@"Failed to create asset extractor"); return 1; @@ -15,7 +15,7 @@ int extractAsset(AAByteStream stream, NSString* outputDirectory) { void* allocated = valloc(ALLOC_SIZE); if (!allocated) { ERRLOG(@"Failed to allocate memory"); - AAAssetExtractorDestroy(extractor); + AAAssetExtractorDestroy(extractor, NULL); return 1; } @@ -36,13 +36,13 @@ int extractAsset(AAByteStream stream, NSString* outputDirectory) { if (written != read) { ERRLOG(@"Data write mismatch: expected %zu, got %zu", read, written); - AAAssetExtractorDestroy(extractor); + AAAssetExtractorDestroy(extractor, NULL); free(allocated); return 1; } } - AAAssetExtractorDestroy(extractor); + AAAssetExtractorDestroy(extractor, NULL); free(allocated); return 0; } diff --git a/src/extract_standalone.m b/src/extract_standalone.m index e10f4be..b55ce6e 100644 --- a/src/extract_standalone.m +++ b/src/extract_standalone.m @@ -1,21 +1,22 @@ #import "extract_standalone.h" + +#import + #import "AppleArchivePrivate.h" +#import "args.h" #import "utils.h" -// TODO: Cleanup -// TODO: Work on filtering - -typedef struct inner_archive_data { +typedef struct nested_archive_data { AAArchiveStream stream; uint64_t size; -}* inner_archive_data; +}* nested_archive_data; -AAByteStream inner_archive_open(AAArchiveStream stream, uint64_t size); -int inner_archive_close(void* stream); -ssize_t inner_archive_read(void* stream, void* buf, size_t nbyte); +AAByteStream nested_archive_open(AAArchiveStream stream, uint64_t size); +int nested_archive_close(void* stream); +ssize_t nested_archive_read(void* stream, void* buf, size_t nbyte); -AAByteStream inner_archive_open(AAArchiveStream stream, uint64_t size) { - inner_archive_data data = malloc(sizeof(struct inner_archive_data)); +AAByteStream nested_archive_open(AAArchiveStream stream, uint64_t size) { + nested_archive_data data = malloc(sizeof(struct nested_archive_data)); if (!data) { return NULL; } @@ -29,15 +30,15 @@ AAByteStream inner_archive_open(AAArchiveStream stream, uint64_t size) { return NULL; } - AACustomByteStreamSetCloseProc(byteStream, inner_archive_close); - AACustomByteStreamSetReadProc(byteStream, inner_archive_read); + AACustomByteStreamSetCloseProc(byteStream, nested_archive_close); + AACustomByteStreamSetReadProc(byteStream, nested_archive_read); AACustomByteStreamSetData(byteStream, data); return byteStream; } -int inner_archive_close(void* stream) { - inner_archive_data data = stream; +int nested_archive_close(void* stream) { + nested_archive_data data = stream; if (data) { free(data); @@ -45,8 +46,8 @@ int inner_archive_close(void* stream) { return 0; } -ssize_t inner_archive_read(void* stream, void* buf, size_t nbyte) { - inner_archive_data data = stream; +ssize_t nested_archive_read(void* stream, void* buf, size_t nbyte) { + nested_archive_data data = stream; size_t size = MIN(nbyte, data->size); if (!size) { @@ -59,7 +60,65 @@ ssize_t inner_archive_read(void* stream, void* buf, size_t nbyte) { return size; } -int extractAssetStandalone(AAByteStream byteStream, NSString* outputDirectory) { +#if DEBUG +static inline NSString* messageToString(AAEntryMessage message) { + NSDictionary* map = @{ + @(AA_ENTRY_MESSAGE_SEARCH_PRUNE_DIR): @"SEARCH_PRUNE_DIR", + @(AA_ENTRY_MESSAGE_SEARCH_EXCLUDE): @"SEARCH_EXCLUDE", + @(AA_ENTRY_MESSAGE_SEARCH_FAIL): @"SEARCH_FAIL", + @(AA_ENTRY_MESSAGE_EXTRACT_BEGIN): @"EXTRACT_BEGIN", + @(AA_ENTRY_MESSAGE_EXTRACT_END): @"EXTRACT_END", + @(AA_ENTRY_MESSAGE_EXTRACT_FAIL): @"EXTRACT_FAIL", + @(AA_ENTRY_MESSAGE_EXTRACT_ATTRIBUTES): @"EXTRACT_ATTRIBUTES", + @(AA_ENTRY_MESSAGE_EXTRACT_XAT): @"EXTRACT_XAT", + @(AA_ENTRY_MESSAGE_EXTRACT_ACL): @"EXTRACT_ACL", + @(AA_ENTRY_MESSAGE_ENCODE_SCANNING): @"ENCODE_SCANNING", + @(AA_ENTRY_MESSAGE_ENCODE_WRITING): @"ENCODE_WRITING", + @(AA_ENTRY_MESSAGE_CONVERT_EXCLUDE): @"CONVERT_EXCLUDE", + @(AA_ENTRY_MESSAGE_PROCESS_EXCLUDE): @"PROCESS_EXCLUDE", + @(AA_ENTRY_MESSAGE_DECODE_READING): @"DECODE_READING" + }; + + return map[@(message)] ? map[@(message)] : @"Unknown"; +} +#endif + +static int aa_callback(void* arg, AAEntryMessage message, const char* path, void* data) { + ExtractionConfiguration* config = (__bridge ExtractionConfiguration*)arg; + + DBGLOG(@"[%@] Message: %@ (%d), Path: %s", config.function, messageToString(message), message, path); + + if (config.regex) { + // TODO: Implement + abort(); + } else if (config.filter) { + int ret = fnmatch(config.filter.UTF8String, path, 0); + DBGLOG(@"[%@] Path: %s, Filter: %@, Ret: %@", config.function, path, config.filter, + ret == 0 ? @"Match" : (ret == FNM_NOMATCH ? @"No match" : @"Error")); + if (ret != 0 && message == AA_ENTRY_MESSAGE_EXTRACT_BEGIN) { + return 1; + } + } + + if (config.list) { + if (message == AA_ENTRY_MESSAGE_EXTRACT_BEGIN) { + // Do not continue extraction + LOG(@"%s", path); + return 1; + } else if (message == AA_ENTRY_MESSAGE_PROCESS_EXCLUDE) { + // This occurs before extraction begins, so we want this to continue + return 0; + } else { + // Skip all other steps of extraction + return 1; + } + } + + // Continue + return 0; +} + +int extractAssetStandalone(AAByteStream byteStream, ExtractionConfiguration* config) { AAArchiveStream decodeStream = AADecodeArchiveInputStreamOpen(byteStream, NULL, NULL, 0, 0); if (!decodeStream) { ERRLOG(@"Failed to open archive decode stream"); @@ -113,7 +172,7 @@ int extractAssetStandalone(AAByteStream byteStream, NSString* outputDirectory) { // maybe label? int lblIndex = AAHeaderGetKeyIndex(header, AA_FIELD_C("LBL")); if (lblIndex == -1) { - ERRLOG(@"Failed to find LBL key index"); + DBGLOG(@"Failed to find LBL key index"); continue; } @@ -132,70 +191,81 @@ int extractAssetStandalone(AAByteStream byteStream, NSString* outputDirectory) { DBGLOG(@"Processing %c entry", (char)yop); if (yop == AA_YOP_TYPE_EXTRACT || yop == AA_YOP_TYPE_DST_FIXUP) { - // TODO: Maybe extract this into a function - AAFieldKeySet keySet = AAFieldKeySetCreate(); + if (config.list && yop == AA_YOP_TYPE_DST_FIXUP) { + DBGLOG(@"Skipping DST_FIXUP entry as we are listing only"); + } else { + // TODO: Maybe extract this into a function + AAFieldKeySet keySet = AAFieldKeySetCreate(); + + int datIndex = AAHeaderGetKeyIndex(header, AA_FIELD_DAT); + if (datIndex == -1) { + ERRLOG(@"Failed to find DAT key index"); + continue; + } + + uint64_t datSize = -1; + uint64_t datOffset = -1; + if (AAHeaderGetFieldBlob(header, datIndex, &datSize, &datOffset) != 0) { + ERRLOG(@"Failed to get DAT"); + continue; + } + + AAByteStream datStream = nested_archive_open(decodeStream, datSize); + if (!datStream) { + ERRLOG(@"Failed to open DAT stream"); + break; + } + + AAByteStream decompressStream = AADecompressionInputStreamOpen(datStream, 0, 0); + if (!decompressStream) { + ERRLOG(@"Failed to open decompress stream"); + AAByteStreamClose(datStream); + break; + } + + AAArchiveStream innerDecodeStream = AADecodeArchiveInputStreamOpen(decompressStream, NULL, NULL, 0, 1); + if (!innerDecodeStream) { + ERRLOG(@"Failed to open inner decode stream"); + AAByteStreamClose(decompressStream); + AAByteStreamClose(datStream); + break; + } + + // TODO: What is the difference between these two? + // TODO: Magic constant + // TODO: CFBridgingRetain will leak + AAArchiveStream extractStream = + yop == AA_YOP_TYPE_DST_FIXUP + ? AAVerifyDirectoryArchiveOutputStreamOpen(config.outputDirectory.UTF8String, keySet, + (void*)CFBridgingRetain([config copyWithFunction:@"VERIFY"]), + aa_callback, UINT64_C(1) << 53, 0) + : AAExtractArchiveOutputStreamOpen(config.outputDirectory.UTF8String, + (void*)CFBridgingRetain([config copyWithFunction:@"EXTRACT"]), aa_callback, 0, + 0); + if (!extractStream) { + ERRLOG(@"Failed to open extract stream"); + AAArchiveStreamClose(innerDecodeStream); + AAByteStreamClose(decompressStream); + AAByteStreamClose(datStream); + break; + } + + if (AAArchiveStreamProcess(innerDecodeStream, extractStream, (void*)CFBridgingRetain([config copyWithFunction:@"PROCESS"]), + aa_callback, 0, 0) < 0) { + ERRLOG(@"Failed to process archive stream"); + AAArchiveStreamClose(extractStream); + AAArchiveStreamClose(innerDecodeStream); + AAByteStreamClose(decompressStream); + AAByteStreamClose(datStream); + break; + } - int datIndex = AAHeaderGetKeyIndex(header, AA_FIELD_DAT); - if (datIndex == -1) { - ERRLOG(@"Failed to find DAT key index"); - continue; - } - - uint64_t datSize = -1; - uint64_t datOffset = -1; - if (AAHeaderGetFieldBlob(header, datIndex, &datSize, &datOffset) != 0) { - ERRLOG(@"Failed to get DAT"); - continue; - } - - AAByteStream datStream = inner_archive_open(decodeStream, datSize); - if (!datStream) { - ERRLOG(@"Failed to open DAT stream"); - break; - } - - AAByteStream decompressStream = AADecompressionInputStreamOpen(datStream, 0, 0); - if (!decompressStream) { - ERRLOG(@"Failed to open decompress stream"); - AAByteStreamClose(datStream); - break; - } - - AAArchiveStream innerDecodeStream = AADecodeArchiveInputStreamOpen(decompressStream, NULL, NULL, 0, 1); - if (!innerDecodeStream) { - ERRLOG(@"Failed to open inner decode stream"); - AAByteStreamClose(decompressStream); - AAByteStreamClose(datStream); - break; - } - - // TODO: What is the difference between these two? - AAArchiveStream extractStream = - yop == AA_YOP_TYPE_DST_FIXUP - ? AAVerifyDirectoryArchiveOutputStreamOpen(outputDirectory.UTF8String, keySet, NULL, NULL, UINT64_C(1) << 53, 0) - : AAExtractArchiveOutputStreamOpen(outputDirectory.UTF8String, NULL, NULL, 0, 0); - if (!extractStream) { - ERRLOG(@"Failed to open extract stream"); - AAArchiveStreamClose(innerDecodeStream); - AAByteStreamClose(decompressStream); - AAByteStreamClose(datStream); - break; - } - - if (AAArchiveStreamProcess(innerDecodeStream, extractStream, NULL, NULL, 0, 0) < 0) { - ERRLOG(@"Failed to process archive stream"); AAArchiveStreamClose(extractStream); AAArchiveStreamClose(innerDecodeStream); AAByteStreamClose(decompressStream); AAByteStreamClose(datStream); - break; + AAFieldKeySetDestroy(keySet); } - - AAArchiveStreamClose(extractStream); - AAArchiveStreamClose(innerDecodeStream); - AAByteStreamClose(decompressStream); - AAByteStreamClose(datStream); - AAFieldKeySetDestroy(keySet); } else { ERRLOG(@"Unknown YOP: %llx", yop); #if DEBUG diff --git a/test_extract.sh b/test_extract.sh index fd1ff31..7dce0c2 100755 --- a/test_extract.sh +++ b/test_extract.sh @@ -1,18 +1,92 @@ #!/usr/bin/env bash set -e +set -o pipefail + +FAILED=0 + +fail() { + echo "$1" + echo "" + if [ "$CI" = "true" ]; then + FAILED=1 + else + exit 1 + fi +} rm -rf tmp -mkdir tmp +mkdir -p tmp for i in tests/*; do i=$(basename "$i") + TEST_DIR="tests/$i" + TMP_DIR="tmp/$i" + echo "Testing $i" - mkdir -p "tmp/$i/a" "tmp/$i/b" - ./aastuff "tests/$i/encrypted.aea" "tmp/$i/a" "$(cat tests/"$i"/key.txt)" - ./aastuff_standalone "tests/$i/encrypted.aea" "tmp/$i/b" "$(cat tests/"$i"/key.txt)" - diff -r "tmp/$i/a" "tmp/$i/b" && echo "Test $i passed" || echo "Test $i failed" + + if [ -f "$TEST_DIR/skip_extract" ]; then + echo "Skipping $i" + echo "" + continue + fi + + mkdir -p "$TMP_DIR/a" "$TMP_DIR/b" + + ret=0 + + if [[ ! -f "$TEST_DIR/expected.txt" ]]; then + fail "Missing expected.txt" + continue + fi + + if [[ ! -f "$TEST_DIR/flags.txt" ]]; then + fail "Missing flags.txt" + continue + fi + + ./aastuff -i "tests/$i/encrypted.aea" -o "tmp/$i/a" --key "$(cat "$TEST_DIR"/expected.txt)" || ret=$? + if [ $ret -ne 0 ]; then + fail "aastuff failed with $ret" + continue + fi + + ./aastuff_standalone -i "tests/$i/encrypted.aea" -o "tmp/$i/b" --key "$(cat "$TEST_DIR"/expected.txt)" || ret=$? + if [ $ret -ne 0 ]; then + fail "aastuff_standalone failed with $ret" + continue + fi + + diff -r "$TMP_DIR/a" "$TMP_DIR/b" || ret=$? + if [ $ret -eq 0 ]; then + echo "Diff passed" + else + fail "Diff failed" + continue + fi + + aa archive -d "tmp/$i/a" -o "tmp/$i/a.aar" -exclude-field all -include-field "$(cat tests/"$i"/flags.txt)" || ret=$? + if [ $ret -ne 0 ]; then + fail "Archive creation failed with $ret" + continue + fi + + aa verify -i "tmp/$i/a.aar" -d "tmp/$i/b" || ret=$? + if [ $ret -eq 0 ]; then + echo "Verify passed" + else + fail "Verify failed" + continue + fi + + echo "Test $i passed" + echo "" done rm -rf tmp echo Done + +if [ $FAILED -ne 0 ]; then + echo "Some tests failed" + exit 1 +fi diff --git a/test_get_key.sh b/test_get_key.sh index fd6d3e5..3a0f65f 100755 --- a/test_get_key.sh +++ b/test_get_key.sh @@ -1,10 +1,20 @@ #!/usr/bin/env bash set -e +set -o pipefail -abort() { +# Use aa list instead of full decryption +FAST=1 +FAILED=0 + +fail() { echo "$1" - exit 1 + echo "" + if [ "$CI" = "true" ]; then + FAILED=1 + else + exit 1 + fi } rm -rf tmp @@ -12,19 +22,69 @@ mkdir tmp for i in tests/*; do i=$(basename "$i") + TEST_DIR="tests/$i" + TMP_DIR="tmp/$i" + echo "Testing $i" - mkdir -p "tmp/$i" + + if [ -e "$TEST_DIR/skip_get_key" ]; then + echo "Skipping $i" + echo "" + continue + fi + + if [ -e "$TEST_DIR/fast_unsupported" ]; then + FAST=0 + fi + + mkdir -p "$TMP_DIR" + + ret=0 + # Ensure expected key is valid first - aea decrypt -i "tests/$i/encrypted.aea" -o "tmp/decrypted" -key-value "base64:$(cat tests/"$i"/expected.txt)" || abort "Failed to decrypt with expected key" + if [ "$FAST" -eq 1 ]; then + aa list -i "$TEST_DIR/encrypted.aea" -key-value "base64:$(cat tests/"$i"/expected.txt)" || ret=$? + else + aea decrypt -i "$TEST_DIR/encrypted.aea" -o "/dev/null" -key-value "base64:$(cat tests/"$i"/expected.txt)" || ret=$? + fi + + if [ $ret -ne 0 ]; then + fail "Failed to decrypt with expected key" + continue + fi + # Get the key - python3 get_key.py "tests/$i/encrypted.aea" > "tmp/$i/actual.txt" || abort "Failed to get key" + python3 get_key.py "$TEST_DIR/encrypted.aea" | tr -d '\n' >"$TMP_DIR/actual.txt" || ret=$? + if [ $ret -ne 0 ]; then + fail "Failed to get key" + continue + fi + # Ensure the key is correct - aea decrypt -i "tests/$i/encrypted.aea" -o "tmp/decrypted" -key-value "base64:$(cat tmp/"$i"/actual.txt)" || abort "Failed to decrypt with actual key" - if ! diff "tmp/$i/actual.txt" "tests/$i/expected.txt"; then + if [ "$FAST" -eq 1 ]; then + aa list -i "$TEST_DIR/encrypted.aea" -key-value "base64:$(cat tmp/"$i"/actual.txt)" || ret=$? + else + aea decrypt -i "$TEST_DIR/encrypted.aea" -o "/dev/null" -key-value "base64:$(cat tmp/"$i"/actual.txt)" || ret=$? + fi + + if [ $ret -ne 0 ]; then + fail "Failed to decrypt with actual key" + continue + fi + + diff "$TMP_DIR/actual.txt" "$TEST_DIR/expected.txt" || ret=$? + if [ $ret -ne 0 ]; then echo "Warning: key does not match expected key, but decryption was successful" fi + echo "Test $i passed" + echo "" done rm -rf tmp echo Done + +if [ $FAILED -ne 0 ]; then + echo "Some tests failed" + exit 1 +fi