Merge pull request #1424 from DarkLordZach/ips-witch

ips_layer: Add support for IPSwitch executable patches

src/common/hex_util.cpp

@@ -18,6 +18,25 @@ u8 ToHexNibble(char c1) {
     return 0;
 }
 
+std::vector<u8> HexStringToVector(std::string_view str, bool little_endian) {
+    std::vector<u8> out(str.size() / 2);
+    if (little_endian) {
+        for (std::size_t i = str.size() - 2; i <= str.size(); i -= 2)
+            out[i / 2] = (ToHexNibble(str[i]) << 4) | ToHexNibble(str[i + 1]);
+    } else {
+        for (std::size_t i = 0; i < str.size(); i += 2)
+            out[i / 2] = (ToHexNibble(str[i]) << 4) | ToHexNibble(str[i + 1]);
+    }
+    return out;
+}
+
+std::string HexVectorToString(const std::vector<u8>& vector, bool upper) {
+    std::string out;
+    for (u8 c : vector)
+        out += fmt::format(upper ? "{:02X}" : "{:02x}", c);
+    return out;
+}
+
 std::array<u8, 16> operator""_array16(const char* str, std::size_t len) {
     if (len != 32) {
         LOG_ERROR(Common,

src/common/hex_util.h

@@ -7,6 +7,7 @@
 #include <array>
 #include <cstddef>
 #include <string>
+#include <vector>
 #include <fmt/format.h>
 #include "common/common_types.h"
 
@@ -14,6 +15,8 @@ namespace Common {
 
 u8 ToHexNibble(char c1);
 
+std::vector<u8> HexStringToVector(std::string_view str, bool little_endian);
+
 template <std::size_t Size, bool le = false>
 std::array<u8, Size> HexStringToArray(std::string_view str) {
     std::array<u8, Size> out{};
@@ -27,6 +30,8 @@ std::array<u8, Size> HexStringToArray(std::string_view str) {
     return out;
 }
 
+std::string HexVectorToString(const std::vector<u8>& vector, bool upper = true);
+
 template <std::size_t Size>
 std::string HexArrayToString(std::array<u8, Size> array, bool upper = true) {
     std::string out;

src/core/file_sys/ips_layer.cpp

@@ -2,7 +2,9 @@
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
+#include <sstream>
 #include "common/assert.h"
+#include "common/hex_util.h"
 #include "common/swap.h"
 #include "core/file_sys/ips_layer.h"
 #include "core/file_sys/vfs_vector.h"
@@ -15,6 +17,12 @@ enum class IPSFileType {
     Error,
 };
 
+constexpr std::array<std::pair<const char*, const char*>, 11> ESCAPE_CHARACTER_MAP{
+    std::pair{"\\a", "\a"}, {"\\b", "\b"},  {"\\f", "\f"},  {"\\n", "\n"},
+    {"\\r", "\r"},          {"\\t", "\t"},  {"\\v", "\v"},  {"\\\\", "\\"},
+    {"\\\'", "\'"},         {"\\\"", "\""}, {"\\\?", "\?"},
+};
+
 static IPSFileType IdentifyMagic(const std::vector<u8>& magic) {
     if (magic.size() != 5)
         return IPSFileType::Error;
@@ -85,4 +93,205 @@ VirtualFile PatchIPS(const VirtualFile& in, const VirtualFile& ips) {
     return std::make_shared<VectorVfsFile>(in_data, in->GetName(), in->GetContainingDirectory());
 }
 
+IPSwitchCompiler::IPSwitchCompiler(VirtualFile patch_text_) : patch_text(std::move(patch_text_)) {
+    Parse();
+}
+
+IPSwitchCompiler::~IPSwitchCompiler() = default;
+
+std::array<u8, 32> IPSwitchCompiler::GetBuildID() const {
+    return nso_build_id;
+}
+
+bool IPSwitchCompiler::IsValid() const {
+    return valid;
+}
+
+static bool StartsWith(std::string_view base, std::string_view check) {
+    return base.size() >= check.size() && base.substr(0, check.size()) == check;
+}
+
+static std::string EscapeStringSequences(std::string in) {
+    for (const auto& seq : ESCAPE_CHARACTER_MAP) {
+        for (auto index = in.find(seq.first); index != std::string::npos;
+             index = in.find(seq.first, index)) {
+            in.replace(index, std::strlen(seq.first), seq.second);
+            index += std::strlen(seq.second);
+        }
+    }
+
+    return in;
+}
+
+void IPSwitchCompiler::ParseFlag(const std::string& line) {
+    if (StartsWith(line, "@flag offset_shift ")) {
+        // Offset Shift Flag
+        offset_shift = std::stoll(line.substr(19), nullptr, 0);
+    } else if (StartsWith(line, "@little-endian")) {
+        // Set values to read as little endian
+        is_little_endian = true;
+    } else if (StartsWith(line, "@big-endian")) {
+        // Set values to read as big endian
+        is_little_endian = false;
+    } else if (StartsWith(line, "@flag print_values")) {
+        // Force printing of applied values
+        print_values = true;
+    }
+}
+
+void IPSwitchCompiler::Parse() {
+    const auto bytes = patch_text->ReadAllBytes();
+    std::stringstream s;
+    s.write(reinterpret_cast<const char*>(bytes.data()), bytes.size());
+
+    std::vector<std::string> lines;
+    std::string stream_line;
+    while (std::getline(s, stream_line)) {
+        // Remove a trailing \r
+        if (!stream_line.empty() && stream_line.back() == '\r')
+            stream_line.pop_back();
+        lines.push_back(std::move(stream_line));
+    }
+
+    for (std::size_t i = 0; i < lines.size(); ++i) {
+        auto line = lines[i];
+
+        // Remove midline comments
+        std::size_t comment_index = std::string::npos;
+        bool within_string = false;
+        for (std::size_t k = 0; k < line.size(); ++k) {
+            if (line[k] == '\"' && (k > 0 && line[k - 1] != '\\')) {
+                within_string = !within_string;
+            } else if (line[k] == '\\' && (k < line.size() - 1 && line[k + 1] == '\\')) {
+                comment_index = k;
+                break;
+            }
+        }
+
+        if (!StartsWith(line, "//") && comment_index != std::string::npos) {
+            last_comment = line.substr(comment_index + 2);
+            line = line.substr(0, comment_index);
+        }
+
+        if (StartsWith(line, "@stop")) {
+            // Force stop
+            break;
+        } else if (StartsWith(line, "@nsobid-")) {
+            // NSO Build ID Specifier
+            auto raw_build_id = line.substr(8);
+            if (raw_build_id.size() != 0x40)
+                raw_build_id.resize(0x40, '0');
+            nso_build_id = Common::HexStringToArray<0x20>(raw_build_id);
+        } else if (StartsWith(line, "#")) {
+            // Mandatory Comment
+            LOG_INFO(Loader, "[IPSwitchCompiler ('{}')] Forced output comment: {}",
+                     patch_text->GetName(), line.substr(1));
+        } else if (StartsWith(line, "//")) {
+            // Normal Comment
+            last_comment = line.substr(2);
+            if (last_comment.find_first_not_of(' ') == std::string::npos)
+                continue;
+            if (last_comment.find_first_not_of(' ') != 0)
+                last_comment = last_comment.substr(last_comment.find_first_not_of(' '));
+        } else if (StartsWith(line, "@enabled") || StartsWith(line, "@disabled")) {
+            // Start of patch
+            const auto enabled = StartsWith(line, "@enabled");
+            if (i == 0)
+                return;
+            LOG_INFO(Loader, "[IPSwitchCompiler ('{}')] Parsing patch '{}' ({})",
+                     patch_text->GetName(), last_comment, line.substr(1));
+
+            IPSwitchPatch patch{last_comment, enabled, {}};
+
+            // Read rest of patch
+            while (true) {
+                if (i + 1 >= lines.size())
+                    break;
+                const auto patch_line = lines[++i];
+
+                // Start of new patch
+                if (StartsWith(patch_line, "@enabled") || StartsWith(patch_line, "@disabled")) {
+                    --i;
+                    break;
+                }
+
+                // Check for a flag
+                if (StartsWith(patch_line, "@")) {
+                    ParseFlag(patch_line);
+                    continue;
+                }
+
+                // 11 - 8 hex digit offset + space + minimum two digit overwrite val
+                if (patch_line.length() < 11)
+                    break;
+                auto offset = std::stoul(patch_line.substr(0, 8), nullptr, 16);
+                offset += offset_shift;
+
+                std::vector<u8> replace;
+                // 9 - first char of replacement val
+                if (patch_line[9] == '\"') {
+                    // string replacement
+                    auto end_index = patch_line.find('\"', 10);
+                    if (end_index == std::string::npos || end_index < 10)
+                        return;
+                    while (patch_line[end_index - 1] == '\\') {
+                        end_index = patch_line.find('\"', end_index + 1);
+                        if (end_index == std::string::npos || end_index < 10)
+                            return;
+                    }
+
+                    auto value = patch_line.substr(10, end_index - 10);
+                    value = EscapeStringSequences(value);
+                    replace.reserve(value.size());
+                    std::copy(value.begin(), value.end(), std::back_inserter(replace));
+                } else {
+                    // hex replacement
+                    const auto value = patch_line.substr(9);
+                    replace.reserve(value.size() / 2);
+                    replace = Common::HexStringToVector(value, is_little_endian);
+                }
+
+                if (print_values) {
+                    LOG_INFO(Loader,
+                             "[IPSwitchCompiler ('{}')]     - Patching value at offset 0x{:08X} "
+                             "with byte string '{}'",
+                             patch_text->GetName(), offset, Common::HexVectorToString(replace));
+                }
+
+                patch.records.insert_or_assign(offset, std::move(replace));
+            }
+
+            patches.push_back(std::move(patch));
+        } else if (StartsWith(line, "@")) {
+            ParseFlag(line);
+        }
+    }
+
+    valid = true;
+}
+
+VirtualFile IPSwitchCompiler::Apply(const VirtualFile& in) const {
+    if (in == nullptr || !valid)
+        return nullptr;
+
+    auto in_data = in->ReadAllBytes();
+
+    for (const auto& patch : patches) {
+        if (!patch.enabled)
+            continue;
+
+        for (const auto& record : patch.records) {
+            if (record.first >= in_data.size())
+                continue;
+            auto replace_size = record.second.size();
+            if (record.first + replace_size > in_data.size())
+                replace_size = in_data.size() - record.first;
+            for (std::size_t i = 0; i < replace_size; ++i)
+                in_data[i + record.first] = record.second[i];
+        }
+    }
+
+    return std::make_shared<VectorVfsFile>(in_data, in->GetName(), in->GetContainingDirectory());
+}
+
 } // namespace FileSys

src/core/file_sys/ips_layer.h

@@ -12,4 +12,34 @@ namespace FileSys {
 
 VirtualFile PatchIPS(const VirtualFile& in, const VirtualFile& ips);
 
+class IPSwitchCompiler {
+public:
+    explicit IPSwitchCompiler(VirtualFile patch_text);
+    ~IPSwitchCompiler();
+
+    std::array<u8, 0x20> GetBuildID() const;
+    bool IsValid() const;
+    VirtualFile Apply(const VirtualFile& in) const;
+
+private:
+    void ParseFlag(const std::string& flag);
+    void Parse();
+
+    bool valid = false;
+
+    struct IPSwitchPatch {
+        std::string name;
+        bool enabled;
+        std::map<u32, std::vector<u8>> records;
+    };
+
+    VirtualFile patch_text;
+    std::vector<IPSwitchPatch> patches;
+    std::array<u8, 0x20> nso_build_id{};
+    bool is_little_endian = false;
+    s64 offset_shift = 0;
+    bool print_values = false;
+    std::string last_comment = "";
+};
+
 } // namespace FileSys

src/core/file_sys/patch_manager.cpp

@@ -73,27 +73,38 @@ VirtualDir PatchManager::PatchExeFS(VirtualDir exefs) const {
     return exefs;
 }
 
-static std::vector<VirtualFile> CollectIPSPatches(const std::vector<VirtualDir>& patch_dirs,
+static std::vector<VirtualFile> CollectPatches(const std::vector<VirtualDir>& patch_dirs,
-                                                  const std::string& build_id) {
+                                               const std::string& build_id) {
-    std::vector<VirtualFile> ips;
+    std::vector<VirtualFile> out;
-    ips.reserve(patch_dirs.size());
+    out.reserve(patch_dirs.size());
     for (const auto& subdir : patch_dirs) {
         auto exefs_dir = subdir->GetSubdirectory("exefs");
         if (exefs_dir != nullptr) {
             for (const auto& file : exefs_dir->GetFiles()) {
-                if (file->GetExtension() != "ips")
+                if (file->GetExtension() == "ips") {
-                    continue;
+                    auto name = file->GetName();
-                auto name = file->GetName();
+                    const auto p1 = name.substr(0, name.find('.'));
-                const auto p1 = name.substr(0, name.find('.'));
+                    const auto this_build_id = p1.substr(0, p1.find_last_not_of('0') + 1);
-                const auto this_build_id = p1.substr(0, p1.find_last_not_of('0') + 1);
 
-                if (build_id == this_build_id)
+                    if (build_id == this_build_id)
-                    ips.push_back(file);
+                        out.push_back(file);
+                } else if (file->GetExtension() == "pchtxt") {
+                    IPSwitchCompiler compiler{file};
+                    if (!compiler.IsValid())
+                        continue;
+
+                    auto this_build_id = Common::HexArrayToString(compiler.GetBuildID());
+                    this_build_id =
+                        this_build_id.substr(0, this_build_id.find_last_not_of('0') + 1);
+
+                    if (build_id == this_build_id)
+                        out.push_back(file);
+                }
             }
         }
     }
 
-    return ips;
+    return out;
 }
 
 std::vector<u8> PatchManager::PatchNSO(const std::vector<u8>& nso) const {
@@ -115,15 +126,24 @@ std::vector<u8> PatchManager::PatchNSO(const std::vector<u8>& nso) const {
     auto patch_dirs = load_dir->GetSubdirectories();
     std::sort(patch_dirs.begin(), patch_dirs.end(),
               [](const VirtualDir& l, const VirtualDir& r) { return l->GetName() < r->GetName(); });
-    const auto ips = CollectIPSPatches(patch_dirs, build_id);
+    const auto patches = CollectPatches(patch_dirs, build_id);
 
     auto out = nso;
-    for (const auto& ips_file : ips) {
+    for (const auto& patch_file : patches) {
-        LOG_INFO(Loader, "    - Appling IPS patch from mod \"{}\"",
+        if (patch_file->GetExtension() == "ips") {
-                 ips_file->GetContainingDirectory()->GetParentDirectory()->GetName());
+            LOG_INFO(Loader, "    - Applying IPS patch from mod \"{}\"",
-        const auto patched = PatchIPS(std::make_shared<VectorVfsFile>(out), ips_file);
+                     patch_file->GetContainingDirectory()->GetParentDirectory()->GetName());
-        if (patched != nullptr)
+            const auto patched = PatchIPS(std::make_shared<VectorVfsFile>(out), patch_file);
-            out = patched->ReadAllBytes();
+            if (patched != nullptr)
+                out = patched->ReadAllBytes();
+        } else if (patch_file->GetExtension() == "pchtxt") {
+            LOG_INFO(Loader, "    - Applying IPSwitch patch from mod \"{}\"",
+                     patch_file->GetContainingDirectory()->GetParentDirectory()->GetName());
+            const IPSwitchCompiler compiler{patch_file};
+            const auto patched = compiler.Apply(std::make_shared<VectorVfsFile>(out));
+            if (patched != nullptr)
+                out = patched->ReadAllBytes();
+        }
     }
 
     if (out.size() < 0x100)
@@ -143,7 +163,7 @@ bool PatchManager::HasNSOPatch(const std::array<u8, 32>& build_id_) const {
     std::sort(patch_dirs.begin(), patch_dirs.end(),
               [](const VirtualDir& l, const VirtualDir& r) { return l->GetName() < r->GetName(); });
 
-    return !CollectIPSPatches(patch_dirs, build_id).empty();
+    return !CollectPatches(patch_dirs, build_id).empty();
 }
 
 static void ApplyLayeredFS(VirtualFile& romfs, u64 title_id, ContentRecordType type) {
@@ -263,8 +283,24 @@ std::map<std::string, std::string, std::less<>> PatchManager::GetPatchVersionNam
     if (mod_dir != nullptr && mod_dir->GetSize() > 0) {
         for (const auto& mod : mod_dir->GetSubdirectories()) {
             std::string types;
-            if (IsDirValidAndNonEmpty(mod->GetSubdirectory("exefs")))
+
-                AppendCommaIfNotEmpty(types, "IPS");
+            const auto exefs_dir = mod->GetSubdirectory("exefs");
+            if (IsDirValidAndNonEmpty(exefs_dir)) {
+                bool ips = false;
+                bool ipswitch = false;
+
+                for (const auto& file : exefs_dir->GetFiles()) {
+                    if (file->GetExtension() == "ips")
+                        ips = true;
+                    else if (file->GetExtension() == "pchtxt")
+                        ipswitch = true;
+                }
+
+                if (ips)
+                    AppendCommaIfNotEmpty(types, "IPS");
+                if (ipswitch)
+                    AppendCommaIfNotEmpty(types, "IPSwitch");
+            }
             if (IsDirValidAndNonEmpty(mod->GetSubdirectory("romfs")))
                 AppendCommaIfNotEmpty(types, "LayeredFS");
 

src/core/file_sys/patch_manager.h

@@ -36,6 +36,7 @@ public:
 
     // Currently tracked NSO patches:
     // - IPS
+    // - IPSwitch
     std::vector<u8> PatchNSO(const std::vector<u8>& nso) const;
 
     // Checks to see if PatchNSO() will have any effect given the NSO's build ID.