Add option to dump SpotBass data (#41)

If you want to help archive SpotPass data before the related Wii U servers are shutdown, please consider doing so by using this separate option in the main menu. Thanks @EpicUsername12 for adding support for this!
dumpling-app · Mar 26, 2024 · f9179f0 · f9179f0
2 parents 85eb42b + 84d6125
commit f9179f0
Show file tree

Hide file tree

Showing 11 changed files with 543 additions and 10 deletions.
diff --git a/Makefile b/Makefile
@@ -77,7 +77,7 @@ CXXFLAGS	:=	$(CFLAGS) -std=c++20
 ASFLAGS		:=	-g $(ARCH)
 LDFLAGS		=	-g $(ARCH) $(RPXSPECS) -Wl,-Map,$(notdir $*.map)
 
-LIBS		:=	-lstdc++ -lwut -lmocha
+LIBS		:=	-lcurl -lmbedtls -lmbedcrypto -lmbedx509 -lz -lstdc++ -lwut -lmocha
 
 #-------------------------------------------------------------------------------
 # list of directories containing libraries, this must be the top level

diff --git a/README.md b/README.md
@@ -25,6 +25,7 @@ You don't need to run/have Mocha CFW or Haxchi, just launch Dumpling from the Ho
 ## How to compile
  - Install [DevkitPro](https://devkitpro.org/wiki/Getting_Started) for your platform.
  - Install freetype2 using DevkitPro's pacman (e.g. `(dkp-)pacman -Sy ppc-pkg-config ppc-freetype`).
+ - Install wiiu-curl using DevkitPro's pacman (e.g. `(dkp-)pacman -Sy wiiu-curl`)
  - Install [wut](https://github.com/devkitpro/wut) through DevkitPro's pacman or compile (and install) the latest source yourself.
  - Compile [libmocha](https://github.com/wiiu-env/libmocha).
  - Then, with all those dependencies installed, you can just run `make` to get the .rpx file that you can run on your Wii U.

diff --git a/source/app/common.h b/source/app/common.h
@@ -188,6 +188,7 @@ struct userAccount {
     bool networkAccount;
     bool passwordCached;
     std::wstring miiName;
+    std::string accountId;
     std::string persistentIdString;
     nn::act::SlotNo slot;
     nn::act::PersistentId persistentId;
@@ -241,7 +242,7 @@ struct titleEntry {
     std::wstring shortTitle;
     std::wstring productCode;
     std::string folderName;
-    
+
     std::optional<titlePart> base;
     std::optional<titlePart> update;
     std::optional<titlePart> dlc;
@@ -250,6 +251,13 @@ struct titleEntry {
     std::optional<filePart> customFile;
 };
 
+struct dumpFileFilter {
+    std::vector<std::string> fileNames;
+    std::vector<std::string> extensions;
+
+    std::vector<std::string> outMatchedFiles;
+};
+
 #if USE_DEBUG_STUBS
 #define IS_CEMU_PRESENT() (OSGetSystemMode() == 0)
 #define USE_WUT_DEVOPTAB() (IS_CEMU_PRESENT() && USE_RAMDISK == 0)

diff --git a/source/app/dumping.cpp b/source/app/dumping.cpp
@@ -7,6 +7,7 @@
 #include "users.h"
 #include "cfw.h"
 #include "gui.h"
+#include "../utils/http.h"
 
 #include "interfaces/transfer.h"
 #include "interfaces/fat32.h"
@@ -362,7 +363,7 @@ bool deleteTitleEntry(titleEntry& entry, dumpingConfig& config, const std::funct
     return true;
 }
 
-bool dumpQueue(std::vector<std::shared_ptr<titleEntry>>& queue, dumpingConfig& config) {
+bool dumpQueue(std::vector<std::shared_ptr<titleEntry>>& queue, dumpingConfig& config, std::optional<dumpFileFilter*> filter) {
     // Delete previously dumped files
     if (config.dumpMethod == DUMP_METHOD::FAT && !USE_WUT_DEVOPTAB()) {
         uint64_t deletedCount = 0;
@@ -473,7 +474,7 @@ bool dumpQueue(std::vector<std::shared_ptr<titleEntry>>& queue, dumpingConfig& c
 
         bool cancelledDumping = false;
         std::filesystem::path currDir;
-        bool dumpSuccess = processTitleEntry(queueEntry, config, [&currDir, &interface, &cancelledDumping, &queueEntry](WALK_EVENT event, const char *filename, const std::string &srcPath, const std::string &destPath) -> bool {
+        bool dumpSuccess = processTitleEntry(queueEntry, config, [&currDir, &interface, &cancelledDumping, &queueEntry, &filter](WALK_EVENT event, const char *filename, const std::string &srcPath, const std::string &destPath) -> bool {
             if (event == WALK_EVENT::MAKE_PATH)
                 return callback_makeDir(interface.get(), destPath, true);
             else if (event == WALK_EVENT::DIR_ENTER) {
@@ -485,8 +486,36 @@ bool dumpQueue(std::vector<std::shared_ptr<titleEntry>>& queue, dumpingConfig& c
                 currDir = currDir.parent_path();
                 return callback_moveDir(interface.get(), currDir.string());
             }
-            else if (event == WALK_EVENT::FILE)
+            else if (event == WALK_EVENT::FILE) {
+
+                if (filter.has_value()) {
+                    auto& fileFilter = filter.value();
+                    bool hasMatched = false;
+
+                    std::string filenameStr(filename);
+                    for (auto& ext : fileFilter->extensions) {
+                        if (filenameStr.ends_with(ext)) {
+                            hasMatched = true;
+                            break;
+                        }
+                    }
+
+                    if (!hasMatched) {
+                        for (auto& name : fileFilter->fileNames) {
+                            if (filenameStr.compare(name) == 0) {
+                                hasMatched = true;
+                                break;
+                            }
+                        }
+                    }
+
+                    if (hasMatched)
+                        fileFilter->outMatchedFiles.push_back(srcPath);
+
+                }
+
                 return callback_copyFile(interface.get(), cancelledDumping, filename, srcPath, destPath);
+            }
             else if (event == WALK_EVENT::BUFFER)
                 return callback_copyBuffer(interface.get(), cancelledDumping, filename, queueEntry.customFile->srcBuffer, queueEntry.customFile->srcBufferSize, destPath+"/"+filename);
             return true;
@@ -672,6 +701,110 @@ void dumpOnlineFiles() {
     else showDialogPrompt(L"Failed to dump the online files...", L"OK");
 }
 
+void dumpSpotpass() {
+    std::vector<std::shared_ptr<titleEntry>> queue;
+    dumpingConfig spotpassConfig = { .dumpTypes = (DUMP_TYPE_FLAGS::CUSTOM) };
+
+    if (!showOptionMenu(spotpassConfig, false)) return;
+
+    titleEntry slcSpotpassDir{ .shortTitle = L"SpotPass Directory", .customFolder = folderPart{.inputPath = convertToPosixPath("/vol/storage_mlc01/usr/save/system/boss"), .outputPath = "/SpotPass Files"} };
+    queue.emplace_back(std::make_shared<titleEntry>(slcSpotpassDir));
+
+    dumpFileFilter filter{ .fileNames = {"task.db"} };
+    if (dumpQueue(queue, spotpassConfig, &filter)) showDialogPrompt(L"Successfully dumped all of the SpotPass files!", L"OK");
+    else showDialogPrompt(L"Failed to dump the SpotPass files...", L"OK");
+
+    wchar_t promptMessage[512];
+    swprintf(promptMessage, 512, L"Do you want to upload the files to the SpotPass Archival Project?\n" 
+                                  "The files do not contain any personally identifiable information.\n"
+                                  "It will upload %d files of 1MB each!\n", filter.outMatchedFiles.size());
+
+    uint8_t doUploadFiles = showDialogPrompt(promptMessage, L"Yes", L"No");
+    if (doUploadFiles == 0) {
+
+        size_t currentUploadIdx = 0;
+        bool taskUploadError = false;
+
+        struct userdata_upload_t {
+            bool* taskUploadError;
+            size_t* currentUploadIdx;
+            dumpFileFilter* filter;
+        } userdata = { &taskUploadError, &currentUploadIdx, &filter };
+
+        auto uploadCallback = [](UploadQueueEntry& entry, float progress) {
+            bool has_started = entry.started;
+            bool has_finished = entry.finished;
+            bool has_error = entry.error;
+
+            userdata_upload_t* userdata = (userdata_upload_t*)entry.userdata;
+            bool& taskUploadError = *userdata->taskUploadError;
+            size_t& currentUploadIdx = *userdata->currentUploadIdx;
+            dumpFileFilter& filter = *userdata->filter;
+
+            if(has_started && has_error) {
+                taskUploadError = true;
+                return;
+            }
+
+            if(has_started && has_finished) {
+                currentUploadIdx++;
+                return;
+            }
+
+            WHBLogFreetypeStartScreen();
+            WHBLogFreetypeScreenPrintBottom(L"===============================");
+            WHBLogFreetypeScreenPrintBottom(L"SpotPass Uploader");
+
+            WHBLogFreetypePrintf(L"Uploading SpotPass task file ... (%d out %d)", currentUploadIdx + 1, filter.outMatchedFiles.size());
+            WHBLogFreetypePrintf(L"File path: %s", filter.outMatchedFiles[currentUploadIdx].c_str());
+            WHBLogFreetypePrint(L"");
+            WHBLogFreetypePrintf(L"Progress: %.1f%%", progress * 100.0f);
+            WHBLogFreetypePrint(L"");
+            if(!has_started && progress <= std::numeric_limits<float>::epsilon()) {
+                WHBLogFreetypePrint(L"Status: Waiting for server to start upload...");
+            } else {
+                WHBLogFreetypePrint(L"Status: Uploading...");
+            }
+
+            WHBLogFreetypeDrawScreen();
+        };
+
+        for (size_t i = 0; i < filter.outMatchedFiles.size(); i++) {
+            std::string filePath = filter.outMatchedFiles[i];
+
+            std::ifstream instream(filePath, std::ios::binary | std::ios::ate);
+            if (!instream) {
+                taskUploadError = true;
+                break;
+            }
+
+            auto fileSize = instream.tellg();
+            instream.seekg(0, std::ios::beg);
+
+            std::vector<uint8_t> data(fileSize);
+            instream.read((char*)data.data(), fileSize);
+
+            http_submitUploadQueue("https://bossarchive.raregamingdump.ca/api/upload/wup", data, uploadCallback, &userdata);
+            WHBLogPrintf("Submitting upload for SpotPass task file ... (%d out %d)", i + 1, filter.outMatchedFiles.size());
+        }
+
+        if (taskUploadError) {
+            showDialogPrompt(L"Failed to read the SpotPass files for upload...", L"OK");
+        }
+
+        while(!taskUploadError && currentUploadIdx < filter.outMatchedFiles.size()) {
+            std::this_thread::yield();
+        }
+
+        if (taskUploadError) {
+            showDialogPrompt(L"Failed to upload the SpotPass files...", L"OK");
+        } else {
+            showDialogPrompt(L"Successfully uploaded all of the SpotPass files!", L"OK");
+        }
+    }
+
+}
+
 void cleanDumpingProcess() {
     WHBLogPrint("Cleaning up after dumping...");
     WHBLogFreetypeDraw();

diff --git a/source/app/dumping.h b/source/app/dumping.h
@@ -1,8 +1,9 @@
 #include "common.h"
 
-bool dumpQueue(std::vector<std::shared_ptr<titleEntry>>& queue, dumpingConfig& config);
+bool dumpQueue(std::vector<std::shared_ptr<titleEntry>>& queue, dumpingConfig& config, std::optional<dumpFileFilter*> filter = std::nullopt);
 bool dumpDisc();
 void dumpMLC();
 void dumpOnlineFiles();
 void dumpAmiibo();
+void dumpSpotpass();
 void cleanDumpingProcess();
diff --git a/source/app/main.cpp b/source/app/main.cpp
@@ -6,6 +6,7 @@
 #include "titles.h"
 #include "users.h"
 #include "gui.h"
+#include "../utils/http.h"
 
 // Initialize correct heaps for CustomRPXLoader
 extern "C" void __init_wut_malloc();
@@ -16,12 +17,12 @@ extern "C" [[maybe_unused]] void __preinit_user(MEMHeapHandle *outMem1, MEMHeapH
 int main() {
     // Initialize libraries
     initializeGUI();
-    WHBLogCafeInit();
     FSInit();
     FSAInit();
     nn::act::Initialize();
     ACPInitialize();
     initializeInputs();
+    http_init();
 
     IMDisableAPD(); // Disable auto-shutdown feature
 
@@ -41,6 +42,7 @@ int main() {
     sleep_for(5s);
 
     // Close application properly
+    http_exit();
     unmountSystemDrives();
     shutdownCFW();
     ACPFinalize();

diff --git a/source/app/menu.cpp b/source/app/menu.cpp
@@ -45,6 +45,8 @@ void showMainMenu() {
         WHBLogFreetypePrintf(L"%C Dump only DLC files of a game", OPTION(6));
         WHBLogFreetypePrintf(L"%C Dump only Save files of a game", OPTION(7));
         WHBLogFreetypePrintf(L"%C Dump whole MLC (everything stored on internal storage)", OPTION(8));
+        WHBLogFreetypePrint(L"");
+        WHBLogFreetypePrintf(L"%C Dump all SpotPass data", OPTION(9));
         WHBLogFreetypeScreenPrintBottom(L"===============================");
         WHBLogFreetypeScreenPrintBottom(L"\uE000 Button = Select Option \uE001 Button = Exit Dumpling");
         WHBLogFreetypeScreenPrintBottom(L"");
@@ -60,7 +62,7 @@ void showMainMenu() {
                 selectedOption--;
                 break;
             }
-            if (navigatedDown() && selectedOption < 8) {
+            if (navigatedDown() && selectedOption < 9) {
                 selectedOption++;
                 break;
             }
@@ -110,6 +112,7 @@ void showMainMenu() {
             dumpMLC();
             break;
         case 9:
+            dumpSpotpass();
             break;
         case 10:
             break;
@@ -146,7 +149,7 @@ bool showOptionMenu(dumpingConfig& config, bool showAccountOption) {
         WHBLogFreetypePrint(L"===============================");
         WHBLogFreetypePrintf(L"%C Dump Destination: %S", OPTION(0), drives.empty() ? L"No Drives Detected" : toWstring(drives[selectedDrive].second).c_str());
         WHBLogFreetypePrintf(L"%C Do Initial Scan For Required Empty Space: %S", OPTION(1), config.scanTitleSize ? L"Yes" : L"No");
-        if (showAccountOption && dumpingOnlineFiles) WHBLogFreetypePrintf(L"%C Online Account: %S", OPTION(2), allUsers[selectedAccount].miiName.c_str());
+        if (showAccountOption && dumpingOnlineFiles) WHBLogFreetypePrintf(L"%C Online Account: %S (NNID: %s)", OPTION(2), allUsers[selectedAccount].miiName.c_str(), allUsers[selectedAccount].networkAccount ? allUsers[selectedAccount].accountId.c_str() : "<NO NNID LINKED>");
         if (showAccountOption && !dumpingOnlineFiles) WHBLogFreetypePrintf(L"%C Account To Get Saves From: %S", OPTION(2), allUsers[selectedAccount].miiName.c_str());
         if (showAccountOption && dumpingOnlineFiles) WHBLogFreetypePrintf(L"%C Merge Account To Default Cemu User: %S", OPTION(3), config.dumpAsDefaultUser ? L"Yes" : L"No");
         if (showAccountOption && !dumpingOnlineFiles) WHBLogFreetypePrintf(L"%C Merge Saves To Default Cemu User: %S", OPTION(3), config.dumpAsDefaultUser ? L"Yes" : L"No");

diff --git a/source/app/users.cpp b/source/app/users.cpp
@@ -13,12 +13,19 @@ bool loadUsers() {
     for (nn::act::SlotNo i=1; i<13; i++) {
         if (nn::act::IsSlotOccupied(i) == true) {
             userAccount newAccount;
-            
+
             // Extract account info
             char16_t miiName[nn::act::MiiNameSize+1];
             nn::Result res = nn::act::GetMiiNameEx((int16_t *)miiName, i);
             if (res.IsFailure()) continue;
 
+            if (nn::act::IsNetworkAccountEx(i)) {
+                char accountId[nn::act::AccountIdSize + 1];
+                res = nn::act::GetAccountIdEx(accountId, i);
+                if (res.IsFailure()) continue;
+                newAccount.accountId = accountId;
+            }
+
             // Convert mii name to normal string
             std::wstring_convert<std::codecvt_utf16<wchar_t>> convert;
             newAccount.miiName = convert.from_bytes(reinterpret_cast<const char*>(miiName), reinterpret_cast<const char*>(miiName + std::char_traits<char16_t>::length(miiName)));