Add NTSC-U, NTSC-J and NTSC-K support (supersedes #104)

.gitattributes

@@ -4,3 +4,12 @@
 # Patch files must stay LF: git apply matches context bytes against LF upstream sources
 *.patch -text
 translator/tests/Translator.Tests/TestAssets/**/*.bin binary
+
+# Generated region tables and headers: reproduced byte-for-byte by the tools in
+# tools/region/ (see its README). The NTSC-U data_addresses.txt is hand-built and not marked.
+projects/mkwii-ntsc-*/MAP.txt linguist-generated=true
+projects/mkwii-ntsc-*/MAP_REPORT.md linguist-generated=true
+projects/mkwii-ntsc-*/region_port.json linguist-generated=true
+projects/mkwii-ntsc-j/data_addresses.txt linguist-generated=true
+projects/mkwii-ntsc-k/data_addresses.txt linguist-generated=true
+runtime/include/region/rmc*.h linguist-generated=true

.gitignore

@@ -65,7 +65,9 @@ project.lock.json
 /.vs/
 /.vscode/
 /.idea/
-/tools/
+/tools/*
+!/tools/region/
+__pycache__/
 /TEMP/
 /tmp/
 /test_output/

Launcher/LocalBuild.ps1

@@ -158,8 +158,48 @@ $toolchainBin = Join-Path $toolchain 'llvm-mingw\bin'
 $cc = Join-Path $toolchainBin 'x86_64-w64-mingw32-clang.exe'
 $cxx = Join-Path $toolchainBin 'x86_64-w64-mingw32-clang++.exe'
 $windres = Join-Path $toolchainBin 'x86_64-w64-mingw32-windres.exe'
-$project = Join-Path $Workspace 'projects\mkwii\recomp.yml'
 $assets = Join-Path $Workspace 'Assets'
+$mainDol = Join-Path $assets 'main.dol'
+$staticRel = Join-Path $assets 'StaticR.rel'
+$bootBin = Join-Path $assets 'DATA\sys\boot.bin'
+
+$project = Join-Path $Workspace 'projects\mkwii\recomp.yml'
+$expectedRelSha = '16d9d146112541fefea701ecb5bc1a496f9d50e4a752fbb5b6778e7c6399f67d'
+$expectedLetter = 'P'
+
+if (Test-Path -LiteralPath $mainDol) {
+    $dolSha = Get-MkwFileSha256 $mainDol
+    if ($dolSha -eq 'd2beec1b1645fcd134efe9e7e63774b546667764ed8d431029daccd725995694') {
+        $project = Join-Path $Workspace 'projects\mkwii-ntsc-u\recomp.yml'
+        $expectedRelSha = '1168107f8fdef27a356df76036db55afe4fbf7752606dbac991726701133a617'
+        $expectedLetter = 'E'
+    } elseif ($dolSha -eq '1b9621ef7c5d97dada103e50e5389730e67f3c2545dda592edd4b5843655af91') {
+        $project = Join-Path $Workspace 'projects\mkwii-ntsc-j\recomp.yml'
+        $expectedRelSha = '88539012d357a1420724e51dc7e351192ce696da4b0045994895518a3fad6fae'
+        $expectedLetter = 'J'
+    } elseif ($dolSha -eq '3098a1e9259b4915e32a4ccb5a1f124823f2a0914db29cd2476e10bdb01a77da') {
+        $project = Join-Path $Workspace 'projects\mkwii-ntsc-k\recomp.yml'
+        $expectedRelSha = 'f441b08e4ccc2d64aadcac8429973f10ea6b1cc8f175500fc084983775f7b3e5'
+        $expectedLetter = 'K'
+    } elseif ($dolSha -ne '80d18895b39c63bd80f457398bfcbb91b7d16ac116a41a88967e954080155b05') {
+        throw "main.dol sha256 ($dolSha) does not match any supported clean Mario Kart Wii release."
+    }
+}
+
+if (Test-Path -LiteralPath $staticRel) {
+    $relSha = Get-MkwFileSha256 $staticRel
+    if ($relSha -cne $expectedRelSha) {
+        throw "StaticR.rel sha256 does not match the clean revision for region $expectedLetter."
+    }
+}
+
+if (Test-Path -LiteralPath $bootBin) {
+    $bytes = [IO.File]::ReadAllBytes($bootBin)
+    $gameId = [Text.Encoding]::ASCII.GetString($bytes, 0, [Math]::Min(6, $bytes.Length))
+    if ($gameId.Length -ge 4 -and $gameId[3] -cne $expectedLetter) {
+        throw "DATA\sys\boot.bin is for disc $gameId, but main.dol is region $expectedLetter. Stale disc assets detected; re-extract your game."
+    }
+}
 $generated = Join-Path $Workspace 'generated'
 $functions = Join-Path $generated 'functions'
 $baseMetadata = Join-Path $generated 'base_translation_output.json'
@@ -304,7 +344,7 @@ try {
 
             Invoke-Checked $translator @(
                 'emit-base-manifest', '--project', $project, '--out', $baseManifestDir,
-                '--functions-dir', $functions, '--translation-output-metadata', $baseMetadata, '--region', 'P'
+                '--functions-dir', $functions, '--translation-output-metadata', $baseMetadata, '--region', $expectedLetter
             ) 'Creating the local base translation manifest' -StepId 'emit-base-manifest'
 
             if (-not [string]::IsNullOrWhiteSpace($TranslationFingerprint)) {
@@ -321,7 +361,7 @@ try {
                 'translate-mod', '--project', $project, '--profile', 'retro-rewind',
                 '--base-manifest', $baseManifest, '--base-translation-output-metadata', $baseMetadata,
                 '--code-pul', $codePul, '--mod-root', $retroRoot, '--mod-name', 'Retro Rewind',
-                '--region', 'P', '--out', $retroOut, '--prefer-cached-inputs', '--emit-cpp',
+                '--region', $expectedLetter, '--out', $retroOut, '--prefer-cached-inputs', '--emit-cpp',
                 '--threads', $translatorThreads
             )
             if ($SkipRetroWfcPayload) {

Launcher/WiiCompiled.Setup.Linux/DiscTool.cs

@@ -11,8 +11,16 @@ namespace WiiCompiled.Setup.Linux;
 /// </summary>
 internal static class DiscTool
 {
-    public static async Task ValidateAndExtractAsync(
-        string isoPath, ProjectManifest manifest, string assetsDirectory, string workspace,
+    private static readonly Dictionary<string, string> ProjectPathsByGameId = new(StringComparer.OrdinalIgnoreCase)
+    {
+        ["RMCP01"] = Path.Combine("projects", "mkwii", "recomp.yml"),
+        ["RMCE01"] = Path.Combine("projects", "mkwii-ntsc-u", "recomp.yml"),
+        ["RMCJ01"] = Path.Combine("projects", "mkwii-ntsc-j", "recomp.yml"),
+        ["RMCK01"] = Path.Combine("projects", "mkwii-ntsc-k", "recomp.yml"),
+    };
+
+    public static async Task<ProjectManifest> ValidateAndExtractAsync(
+        string isoPath, string assetsDirectory, string workspace,
         string? nodToolBin, IInstallReporter reporter, CancellationToken cancellationToken)
     {
         var nodTool = nodToolBin ?? await NodToolProvider.ResolveAsync(workspace, cancellationToken);
@@ -23,12 +31,13 @@ public static async Task ValidateAndExtractAsync(
         // matching the original dolphin-tool `header` step this replaces.
         reporter.Progress(InstallStages.ExtractDisc, "Reading the disc header", 2);
         var info = NodToolInfoParser.Parse(await RunInfoAsync(nodTool, isoPath, cancellationToken));
-        if (!string.Equals(info.GameId, manifest.GameId, StringComparison.Ordinal))
+        if (!ProjectPathsByGameId.TryGetValue(info.GameId, out var projectRelativePath))
         {
             throw new InvalidOperationException(
-                $"This disc is '{info.GameId}', not the expected '{manifest.GameId}' (Mario Kart Wii, region {manifest.Region}). " +
-                "Only your own legally-owned copy of that exact game/region can be used.");
+                $"This disc is '{info.GameId}', which is not a supported Mario Kart Wii release (supported: RMCP01, RMCE01, RMCJ01, RMCK01). " +
+                "Only your own legally-owned copy of a supported game/region can be used.");
         }
+        var manifest = ProjectManifest.Load(Path.Combine(workspace, projectRelativePath));
 
         // Extracted straight into Assets/DATA (kept, not a scratch dir) - the runtime reads course/
         // texture/audio data from this directory live via [paths] dvd_root, not just at translation
@@ -45,23 +54,73 @@ public static async Task ValidateAndExtractAsync(
 
         var dolSha = Sha256Of(dolPath);
         var relSha = Sha256Of(relPath);
-        if (!string.Equals(dolSha, manifest.DolSha256, StringComparison.Ordinal))
+        if (!string.Equals(dolSha, manifest.DolSha256, StringComparison.OrdinalIgnoreCase))
         {
             throw new InvalidOperationException(
-                $"main.dol sha256 mismatch: expected {manifest.DolSha256}, got {dolSha}. " +
+                $"main.dol sha256 mismatch for {manifest.GameId}: expected {manifest.DolSha256}, got {dolSha}. " +
                 "This disc revision does not match what this project's manifest is pinned to.");
         }
-        if (!string.Equals(relSha, manifest.RelSha256, StringComparison.Ordinal))
+        if (!string.Equals(relSha, manifest.RelSha256, StringComparison.OrdinalIgnoreCase))
         {
             throw new InvalidOperationException(
-                $"StaticR.rel sha256 mismatch: expected {manifest.RelSha256}, got {relSha}. " +
+                $"StaticR.rel sha256 mismatch for {manifest.GameId}: expected {manifest.RelSha256}, got {relSha}. " +
                 "This disc revision does not match what this project's manifest is pinned to.");
         }
 
         Directory.CreateDirectory(assetsDirectory);
         File.Copy(dolPath, Path.Combine(assetsDirectory, "main.dol"), overwrite: true);
         File.Copy(relPath, Path.Combine(assetsDirectory, "StaticR.rel"), overwrite: true);
         reporter.Progress(InstallStages.ExtractDisc, "Disc validated and extracted", 6);
+        return manifest;
+    }
+
+    public static ProjectManifest ResolveProjectManifestForAssets(string assetsDirectory, string workspace)
+    {
+        var dolPath = Path.Combine(assetsDirectory, "main.dol");
+        if (!File.Exists(dolPath)) throw new FileNotFoundException("main.dol is missing from Assets", dolPath);
+        var relPath = Path.Combine(assetsDirectory, "StaticR.rel");
+        if (!File.Exists(relPath)) throw new FileNotFoundException("StaticR.rel is missing from Assets", relPath);
+
+        var dolSha = Sha256Of(dolPath);
+        ProjectManifest? manifest = null;
+        foreach (var projectRelPath in ProjectPathsByGameId.Values)
+        {
+            var candidate = ProjectManifest.Load(Path.Combine(workspace, projectRelPath));
+            if (string.Equals(candidate.DolSha256, dolSha, StringComparison.OrdinalIgnoreCase))
+            {
+                manifest = candidate;
+                break;
+            }
+        }
+        if (manifest is null)
+        {
+            throw new InvalidOperationException(
+                $"Assets/main.dol sha256 ({dolSha}) does not match any supported clean Mario Kart Wii release.");
+        }
+
+        var relSha = Sha256Of(relPath);
+        if (!string.Equals(relSha, manifest.RelSha256, StringComparison.OrdinalIgnoreCase))
+        {
+            throw new InvalidOperationException(
+                $"Assets/StaticR.rel sha256 mismatch for {manifest.GameId}: expected {manifest.RelSha256}, got {relSha}.");
+        }
+
+        var bootBin = Path.Combine(assetsDirectory, "DATA", "sys", "boot.bin");
+        if (File.Exists(bootBin))
+        {
+            using var stream = File.OpenRead(bootBin);
+            var buffer = new byte[6];
+            var read = stream.Read(buffer, 0, 6);
+            var gameId = System.Text.Encoding.ASCII.GetString(buffer, 0, read);
+            if (gameId.Length >= 4 && !gameId.StartsWith("RMC" + manifest.Region, StringComparison.OrdinalIgnoreCase))
+            {
+                throw new InvalidOperationException(
+                    $"Assets/DATA/sys/boot.bin is for disc '{gameId}', but main.dol is {manifest.GameId}. " +
+                    "Stale disc assets detected; re-extract your game.");
+            }
+        }
+
+        return manifest;
     }
 
     private static async Task<string> RunInfoAsync(string nodTool, string isoPath, CancellationToken cancellationToken)

Launcher/WiiCompiled.Setup.Linux/Program.cs

@@ -95,13 +95,13 @@ private static async Task InstallAsync(Dictionary<string, string?> flags, IInsta
         if (installsRetro) retroDir = RetroRewindSource.ResolveRetroRewind6(retroDir!);
 
         var workspace = flags.GetValueOrDefault("workspace") ?? WorkspaceLocator.FindFrom(AppContext.BaseDirectory);
-        var manifest = ProjectManifest.Load(Path.Combine(workspace, "projects", "mkwii", "recomp.yml"));
         var assetsDir = Path.Combine(workspace, "Assets");
 
         reporter.Progress(InstallStages.Validate, "Checking prerequisites", 1);
+        ProjectManifest manifest;
         if (flags.TryGetValue("game", out var isoPath) && !string.IsNullOrEmpty(isoPath))
         {
-            await DiscTool.ValidateAndExtractAsync(isoPath, manifest, assetsDir, workspace,
+            manifest = await DiscTool.ValidateAndExtractAsync(isoPath, assetsDir, workspace,
                 flags.GetValueOrDefault("disc-tool-bin"), reporter, token);
         }
         else
@@ -114,6 +114,7 @@ await DiscTool.ValidateAndExtractAsync(isoPath, manifest, assetsDir, workspace,
                     "No --game ISO was given and Assets/main.dol + Assets/StaticR.rel are not already present. " +
                     "Either pass --game <path-to-iso>, or extract them yourself first (see translator/README.md).");
             }
+            manifest = DiscTool.ResolveProjectManifestForAssets(assetsDir, workspace);
         }
 
         var state = JsonState.TryRead<InstallState>(StatePath) ?? new InstallState { Workspace = workspace };

Launcher/WiiCompiled.Setup.Windows/InstallerEngine.cs

@@ -509,7 +509,7 @@ private static void ValidateExtractedGame(string dataRoot, PayloadManifest manif
             !relHash.Equals(manifest.ExpectedRelSha256, StringComparison.OrdinalIgnoreCase))
         {
             throw new InvalidDataException(
-                "The disc is RMCP01 but does not match the supported clean PAL revision. " +
+                $"The disc is {manifest.ExpectedGameId} but does not match the supported clean revision. " +
                 "Patched or otherwise modified game code cannot be installed safely.");
         }
     }

Launcher/local-build-macos.command

@@ -18,7 +18,7 @@ Usage: local-build-macos.command --output-dir DIR [options]
   --profile {base|retro-rewind|both}  Product to build (default: base)
   --output-dir DIR                Output .app directory (required; Retro Rewind for both)
   --base-output-dir DIR           Base .app directory (required with --profile both)
-  --game IMAGE --nodtool PATH     Extract and verify a clean PAL RMCP01 disc image first
+  --game IMAGE --nodtool PATH     Extract and verify a clean Mario Kart Wii disc image first
   --retro-rewind-package-dir DIR  RetroRewind6 directory (required for Retro Rewind)
   --retro-wfc-offline-dir DIR     Directory containing binary/payload.RMCPD00.bin
   --skip-retro-wfc-payload        Build Retro Rewind without the shared Retro-WFC payload
@@ -71,13 +71,50 @@ if (( builds_retro )); then
 fi
 for tool in "$cmake_bin" "$ninja_bin" clang clang++ shasum; do command -v "$tool" >/dev/null || fail "required tool not found: $tool"; done
 
-project="$workspace/projects/mkwii/recomp.yml"; assets="$workspace/Assets"; generated="$workspace/generated"
+assets="$workspace/Assets"; generated="$workspace/generated"
 functions="$generated/functions"; metadata="$generated/base_translation_output.json"; manifest_dir="$workspace/build/base"
 manifest="$manifest_dir/mkwii_base_manifest.json"; shards="$generated/build_shards"; native_build="$workspace/native-build-macos"
-assert_file "$project" 'translation project'
 if [[ -n "$game" ]]; then "$script_dir/macos/extract-disc.command" --game "$game" --assets-dir "$assets" --nodtool "$nodtool"; fi
 assert_file "$assets/main.dol" 'extracted main.dol'; assert_file "$assets/StaticR.rel" 'extracted StaticR.rel'
 
+dol_hash=$(sha256 "$assets/main.dol")
+case "$dol_hash" in
+    d2beec1b1645fcd134efe9e7e63774b546667764ed8d431029daccd725995694)
+        project="$workspace/projects/mkwii-ntsc-u/recomp.yml"
+        expected_rel="1168107f8fdef27a356df76036db55afe4fbf7752606dbac991726701133a617"
+        expected_letter="E"
+        ;;
+    1b9621ef7c5d97dada103e50e5389730e67f3c2545dda592edd4b5843655af91)
+        project="$workspace/projects/mkwii-ntsc-j/recomp.yml"
+        expected_rel="88539012d357a1420724e51dc7e351192ce696da4b0045994895518a3fad6fae"
+        expected_letter="J"
+        ;;
+    3098a1e9259b4915e32a4ccb5a1f124823f2a0914db29cd2476e10bdb01a77da)
+        project="$workspace/projects/mkwii-ntsc-k/recomp.yml"
+        expected_rel="f441b08e4ccc2d64aadcac8429973f10ea6b1cc8f175500fc084983775f7b3e5"
+        expected_letter="K"
+        ;;
+    80d18895b39c63bd80f457398bfcbb91b7d16ac116a41a88967e954080155b05)
+        project="$workspace/projects/mkwii/recomp.yml"
+        expected_rel="16d9d146112541fefea701ecb5bc1a496f9d50e4a752fbb5b6778e7c6399f67d"
+        expected_letter="P"
+        ;;
+    *)
+        fail "Assets/main.dol sha256 ($dol_hash) does not match any supported clean Mario Kart Wii release"
+        ;;
+esac
+
+rel_hash=$(sha256 "$assets/StaticR.rel")
+[[ "$rel_hash" == "$expected_rel" ]] || fail "Assets/StaticR.rel sha256 does not match the clean revision for region $expected_letter"
+
+if [[ -f "$assets/DATA/sys/boot.bin" ]]; then
+    boot_id=$(head -c 6 "$assets/DATA/sys/boot.bin" 2>/dev/null || true)
+    if [[ ${#boot_id} -ge 4 && "${boot_id:3:1}" != "$expected_letter" ]]; then
+        fail "Assets/DATA/sys/boot.bin is for disc $boot_id, but main.dol is region $expected_letter. Stale disc assets detected; re-extract your game."
+    fi
+fi
+assert_file "$project" 'translation project'
+
 if (( force_clean )); then
     step force-clean 'Discarding translation and native build caches'
     rm -rf "$generated" "$manifest_dir" "$native_build"
@@ -119,9 +156,9 @@ step translate-base 'Translating the user-owned base game'
 rm -rf "$functions" "$metadata" "$manifest_dir"; mkdir -p "$functions" "$manifest_dir"
 translator translate-recursive "$entry_point" --project "$project" --outdir "$functions" --output-metadata "$metadata" --production-source-bundle "$generated/base_translation_sources.bin" --no-function-files --prune-stale --threads "$translator_threads"
 step emit-base-manifest 'Creating the local base translation manifest'
-translator emit-base-manifest --project "$project" --out "$manifest_dir" --functions-dir "$functions" --translation-output-metadata "$metadata" --region P
+translator emit-base-manifest --project "$project" --out "$manifest_dir" --functions-dir "$functions" --translation-output-metadata "$metadata" --region "$expected_letter"
 if (( builds_retro )); then
-    mod_out="$workspace/build/mods/retro_rewind_full_cpp"; args=(translate-mod --project "$project" --profile retro-rewind --base-manifest "$manifest" --base-translation-output-metadata "$metadata" --code-pul "$retro_root/Binaries/Code.pul" --mod-root "$retro_root" --mod-name 'Retro Rewind' --region P --out "$mod_out" --prefer-cached-inputs --emit-cpp --threads "$translator_threads")
+    mod_out="$workspace/build/mods/retro_rewind_full_cpp"; args=(translate-mod --project "$project" --profile retro-rewind --base-manifest "$manifest" --base-translation-output-metadata "$metadata" --code-pul "$retro_root/Binaries/Code.pul" --mod-root "$retro_root" --mod-name 'Retro Rewind' --region "$expected_letter" --out "$mod_out" --prefer-cached-inputs --emit-cpp --threads "$translator_threads")
     if (( skip_retro_wfc )); then args+=(--skip-retro-wfc); else offline_payload="$retro_wfc/binary/payload.RMCPD00.bin"; assert_file "$offline_payload" 'Offline Retro-WFC payload'; args+=(--retro-wfc-payload "$offline_payload"); fi
     step translate-mod 'Translating Retro Rewind'; translator "${args[@]}"
 fi