dat: fix symbol ordering, format generated source in-place

.nix/objdiff.nix

@@ -1,25 +1,16 @@
 {
-  stdenvNoCC,
   lib,
   fontconfig,
   pkg-config,
   rustPlatform,
-  srcOnly,
   src,
 }:
 
 rustPlatform.buildRustPackage (finalAttrs: {
   pname = "objdiff";
   version = src.shortRev;
 
-  src = srcOnly {
-    name = "objdiff-patched";
-    inherit src;
-    stdenv = stdenvNoCC;
-    patches = [
-      ./duplicate-similar-dep.patch
-    ];
-  };
+  inherit src;
 
   cargoBuildFlags = [
     "--workspace"

CMakePresets.json

@@ -42,8 +42,7 @@
         "MELEE_DWARF": {
           "type": "BOOL",
           "value": true
-        },
-        "NEWLIB_INCLUDE": "$env{NEWLIB_INCLUDE}"
+        }
       }
     },
     {

cmake/Dat.cmake

@@ -1,18 +1,43 @@
 # Samples of the .dat archives' data, typed by the DWARF build and compared
 # with objdiff (see tools/dat-cli). Each archive is a unit, built in four
-# steps: slice (archive → target/<unit>.o), codegen (→ gen/<unit>.c, which
-# includes a header and source per root in gen/<unit>/), format (→ src/)
+# steps: slice (archive → target/<unit>.o), codegen (→ src/<unit>.c, which
+# includes a header and source per root in src/<unit>/), format (in place)
 # and compile (→ base/<unit>.o). The build directory is also the objdiff
 # project.
 include_guard(GLOBAL)
 
+# Configured through a symlink, the build names the same files by two paths
+# (the physical one CMake resolves, and the logical one from $PWD), and
+# ninja then reruns CMake and rebuilds everything on every build
+file(REAL_PATH "${CMAKE_SOURCE_DIR}" _dat_real_source)
+if(NOT _dat_real_source STREQUAL CMAKE_SOURCE_DIR)
+    message(FATAL_ERROR "Configure from ${_dat_real_source}, not through a symlink (${CMAKE_SOURCE_DIR}): cd -P there first")
+endif()
+
 if(NOT MELEE_DWARF)
     message(FATAL_ERROR "MELEE_DAT_SAMPLES needs MELEE_DWARF: the samples are typed by its DWARF")
 endif()
 
 set(MELEE_DAT "" CACHE FILEPATH "melee-dat binary; built with cargo if empty")
 set(MELEE_DAT_FILES "${CMAKE_SOURCE_DIR}/orig/${MELEE_VERSION}/files"
     CACHE PATH "The game's files, with its .dat archives")
+# What to sample is the user's choice, not the project's
+set(MELEE_DAT_SAMPLES_ALL "" CACHE STRING
+    "Archives (globs, e.g. PlFx.dat;Gr*.dat) to sample every typed object of, not one instance per type")
+set(MELEE_DAT_SAMPLES_EXCLUDE "" CACHE STRING
+    "Types (globs on their names) never to sample, e.g. bulky vertex or image records")
+
+set(_dat_all_regexes)
+foreach(_glob IN LISTS MELEE_DAT_SAMPLES_ALL)
+    string(REPLACE "." "\\." _regex "${_glob}")
+    string(REPLACE "*" ".*" _regex "${_regex}")
+    string(REPLACE "?" "." _regex "${_regex}")
+    list(APPEND _dat_all_regexes "^${_regex}$")
+endforeach()
+set(_dat_exclude)
+foreach(_glob IN LISTS MELEE_DAT_SAMPLES_EXCLUDE)
+    list(APPEND _dat_exclude --exclude "${_glob}")
+endforeach()
 
 set(_dat_config "${CMAKE_SOURCE_DIR}/config/${MELEE_VERSION}/dat.yml")
 set(_dat_symbols "${CMAKE_SOURCE_DIR}/config/${MELEE_VERSION}/dat_symbols.txt")
@@ -27,9 +52,10 @@ if(MELEE_DAT)
     set(_dat_tool "${MELEE_DAT}")
 else()
     set(_dat_tool "${CMAKE_CURRENT_BINARY_DIR}/cargo/release/melee-dat")
+    find_program(MELEE_CARGO cargo REQUIRED)
     add_custom_command(
         OUTPUT "${_dat_tool}"
-        COMMAND cargo build --release -p melee-dat
+        COMMAND "${MELEE_CARGO}" build --release -p melee-dat
             --manifest-path "${CMAKE_SOURCE_DIR}/Cargo.toml"
             --target-dir "${CMAKE_CURRENT_BINARY_DIR}/cargo"
         DEPFILE "${_dat_tool}.d"
@@ -69,46 +95,61 @@ foreach(_archive IN LISTS _dat_archives)
     get_filename_component(_unit "${_archive}" NAME_WE)
     set(_target "target/${_unit}.o")
     set(_sidecar "target/${_unit}.samples")
-    set(_generated "gen/${_unit}.c")
+    set(_layout "target/${_unit}.ld")
+    set(_object "obj/${_unit}.o")
     set(_source "src/${_unit}.c")
+    set(_formatted "stamp/${_unit}.formatted")
     set(_base "base/${_unit}.o")
 
+    set(_all)
+    foreach(_regex IN LISTS _dat_all_regexes)
+        if(_file MATCHES "${_regex}")
+            set(_all --all)
+        endif()
+    endforeach()
     add_custom_command(
-        OUTPUT "${_target}" "${_sidecar}"
+        OUTPUT "${_target}" "${_sidecar}" "${_layout}"
         COMMAND "${_dat_tool}" samples slice "${_file}" "${_dat_config}"
             -p "${CMAKE_SOURCE_DIR}" --types types.bin
-            --files "${MELEE_DAT_FILES}" -o "${_target}"
+            --files "${MELEE_DAT_FILES}" -o "${_target}" ${_all} ${_dat_exclude}
         DEPENDS "${_archive}" types.bin "${_dat_tool}" "${_dat_config}"
             "${_dat_symbols}"
         COMMENT "Slicing ${_file}"
         VERBATIM
     )
     add_custom_command(
-        OUTPUT "${_generated}"
+        OUTPUT "${_source}"
         COMMAND "${_dat_tool}" samples codegen "${_target}" --types types.bin
-            -o "${_generated}"
+            -o "${_source}"
         DEPENDS "${_target}" "${_sidecar}" types.bin "${_dat_tool}"
-        COMMENT "Generating ${_generated}"
+        COMMENT "Generating ${_source}"
         VERBATIM
     )
     add_custom_command(
-        OUTPUT "${_source}"
+        OUTPUT "${_formatted}"
         COMMAND "${CMAKE_COMMAND}"
             "-DCLANG_FORMAT=${MELEE_CLANG_FORMAT}"
             "-DSTYLE=${CMAKE_SOURCE_DIR}/.clang-format"
-            "-DGENERATED=${CMAKE_CURRENT_BINARY_DIR}/${_generated}"
             "-DSOURCE=${CMAKE_CURRENT_BINARY_DIR}/${_source}"
+            "-DSTAMP=${CMAKE_CURRENT_BINARY_DIR}/${_formatted}"
             -P "${CMAKE_SOURCE_DIR}/cmake/DatFormat.cmake"
-        DEPENDS "${_generated}" "${CMAKE_SOURCE_DIR}/.clang-format"
+        DEPENDS "${_source}" "${CMAKE_SOURCE_DIR}/.clang-format"
             "${CMAKE_SOURCE_DIR}/cmake/DatFormat.cmake"
         COMMENT "Formatting ${_source}"
         VERBATIM
     )
     add_custom_command(
         OUTPUT "${_base}"
-        COMMAND "${CMAKE_C_COMPILER}" "@${_dat_flags}" -MD -MF "${_base}.d"
-            -c "${_source}" -o "${_base}"
-        DEPENDS "${_source}" "${_dat_flags}"
+        BYPRODUCTS "${_object}"
+        # Each sample in its own section, then linked into .data in the
+        # target's order: clang lays variables out where an initializer
+        # first points to them
+        COMMAND "${CMAKE_C_COMPILER}" "@${_dat_flags}" -fdata-sections
+            -MD -MF "${_base}.d" -MT "${_base}"
+            -c "${_source}" -o "${_object}"
+        COMMAND "${CMAKE_LINKER}" -r -T "${_layout}" "${_object}"
+            -o "${_base}"
+        DEPENDS "${_source}" "${_formatted}" "${_layout}" "${_dat_flags}"
         DEPFILE "${_base}.d"
         COMMENT "Compiling ${_source}"
         VERBATIM

cmake/DatFormat.cmake

@@ -1,21 +1,17 @@
-# Copies a unit's generated C (gen/<unit>.c and gen/<unit>/) into src/ and
-# formats it; run by Dat.cmake's format step with -P.
+# Formats a unit's generated C (src/<unit>.c and src/<unit>/) in place, then
+# writes a stamp; run by Dat.cmake's format step with -P.
 #   CLANG_FORMAT  clang-format
 #   STYLE         the .clang-format file
-#   GENERATED     gen/<unit>.c
 #   SOURCE        src/<unit>.c
+#   STAMP         stamp/<unit>.formatted, written after formatting so that
+#                 it is newer than the sources
 cmake_minimum_required(VERSION 3.20)
 
-cmake_path(REMOVE_EXTENSION GENERATED LAST_ONLY OUTPUT_VARIABLE _gen_dir)
-cmake_path(REMOVE_EXTENSION SOURCE LAST_ONLY OUTPUT_VARIABLE _src_dir)
-cmake_path(GET SOURCE PARENT_PATH _src_parent)
-
-# The roots of the previous generation may be gone
-file(REMOVE_RECURSE "${_src_dir}")
-file(COPY "${_gen_dir}" "${GENERATED}" DESTINATION "${_src_parent}")
-file(GLOB _files "${_src_dir}/*.[ch]")
+cmake_path(REMOVE_EXTENSION SOURCE LAST_ONLY OUTPUT_VARIABLE _dir)
+file(GLOB _files "${_dir}/*.[ch]")
 
 execute_process(
     COMMAND "${CLANG_FORMAT}" "--style=file:${STYLE}" -i "${SOURCE}" ${_files}
     COMMAND_ERROR_IS_FATAL ANY
 )
+file(TOUCH "${STAMP}")

cmake/Dwarf.cmake

@@ -14,6 +14,10 @@ if(MELEE_VERSION_NUM EQUAL -1)
     message(FATAL_ERROR "Unknown MELEE_VERSION ${MELEE_VERSION}; one of: ${MELEE_VERSIONS}")
 endif()
 
+if(NOT NEWLIB_INCLUDE)
+    message(FATAL_ERROR "NEWLIB_INCLUDE is not set: configure from the dev shell")
+endif()
+
 target_compile_definitions(melee PRIVATE
     LINT
     DAT_ANNOTATIONS

cmake/MeleeConfig.cmake

@@ -3,9 +3,18 @@ include_guard(GLOBAL)
 
 get_filename_component(_melee_root "${CMAKE_CURRENT_LIST_DIR}/.." ABSOLUTE)
 
+# From the dev shell's environment when it has it, else as cached: ninja
+# re-runs CMake from wherever it is started (e.g. objdiff)
+if(DEFINED ENV{AURORA_SRC})
+    set(AURORA_SRC "$ENV{AURORA_SRC}" CACHE PATH "Aurora's source" FORCE)
+endif()
+if(NOT AURORA_SRC)
+    message(FATAL_ERROR "AURORA_SRC is not set: configure from the dev shell")
+endif()
+
 add_library(melee_game_headers INTERFACE)
 target_include_directories(melee_game_headers INTERFACE
-    $ENV{AURORA_SRC}/include
+    ${AURORA_SRC}/include
     ${_melee_root}/src
     ${_melee_root}/libs/doldecomp/include
 )

cmake/ppc32-clang.cmake

@@ -5,14 +5,25 @@ set(CMAKE_SYSTEM_PROCESSOR powerpc)
 
 set(CMAKE_C_COMPILER clang)
 set(CMAKE_C_COMPILER_TARGET ppc32-none-eabi)
-set(CMAKE_AR llvm-ar)
-set(CMAKE_RANLIB llvm-ranlib)
-set(CMAKE_LINKER ld.lld)
+# By absolute path, so that the build doesn't need the dev shell's PATH
+find_program(MELEE_LLVM_AR llvm-ar REQUIRED)
+find_program(MELEE_LLVM_RANLIB llvm-ranlib REQUIRED)
+find_program(MELEE_LLD ld.lld REQUIRED)
+set(CMAKE_AR "${MELEE_LLVM_AR}")
+set(CMAKE_RANLIB "${MELEE_LLVM_RANLIB}")
+set(CMAKE_LINKER "${MELEE_LLD}")
 
 # There is no C runtime to link a test executable against
 set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
 
-set(NEWLIB_INCLUDE "" CACHE PATH "newlib include directory for powerpc-none-eabi")
+# From the dev shell's environment when it has it, else as cached: ninja
+# re-runs CMake from wherever it is started (e.g. objdiff)
+set(_doc "newlib include directory for powerpc-none-eabi")
+if(DEFINED ENV{NEWLIB_INCLUDE})
+    set(NEWLIB_INCLUDE "$ENV{NEWLIB_INCLUDE}" CACHE PATH "${_doc}" FORCE)
+else()
+    set(NEWLIB_INCLUDE "" CACHE PATH "${_doc}")
+endif()
 if(NEWLIB_INCLUDE)
     set(CMAKE_C_STANDARD_INCLUDE_DIRECTORIES "${NEWLIB_INCLUDE}")
 endif()

flake.nix

@@ -19,7 +19,7 @@
       flake = false;
     };
     objdiff = {
-      url = "github:encounter/objdiff/v3.6.1";
+      url = "github:encounter/objdiff/v3.8.2";
       flake = false;
     };
     sjiswrap = {

src/sysdolphin/baselib/jobj.h

@@ -724,7 +724,6 @@ static inline void HSD_JObjRefThis(HSD_JObj* jobj)
 
 void HSD_JObjResolveRefs(HSD_JObj* jobj, HSD_Joint* joint);
 void HSD_JObjUnrefThis(HSD_JObj* jobj);
-void HSD_JObjRefThis(HSD_JObj* jobj);
 void HSD_JObjMakeMatrix(HSD_JObj* jobj);
 void RecalcParentTrspBits(HSD_JObj* jobj);
 void HSD_JObjAddChild(HSD_JObj* jobj, HSD_JObj* child);

tools/dat-cli/README.md

@@ -127,9 +127,13 @@ melee-dat samples report build/GALE01/dat
   the other data they point to, and designated initializers generated from
   the types; pointers into other roots include those roots' headers
 - `src/<archive>.c`: the unit, which includes every root's source; all of
-  `src` is generated into `gen` and formatted with the repository's
-  `.clang-format`
-- `base/<archive>.o`: that C, compiled with the DWARF build's flags
+  it is formatted in place with the repository's `.clang-format`
+  (`stamp/<archive>.formatted` records that)
+- `base/<archive>.o`: that C, compiled with the DWARF build's flags, one
+  section per variable (`obj/<archive>.o`), then linked with
+  `target/<archive>.ld` into one `.data` in the target's order (clang lays
+  variables out where they are first pointed to, not where they are
+  defined)
 
 `compile_commands.json` there gives clangd the same flags as the build.
 
@@ -144,6 +148,13 @@ that doesn't round-trip. A union is written through the member its tag
 chose; a union object is declared as that member (`typeof(((union U *)
 0)->member)`), since the archive only holds that member's bytes.
 
+What to sample is up to you, in the build's cache: by default each archive
+gives its best instance of each type. `MELEE_DAT_SAMPLES_ALL` takes archive
+globs whose every typed object becomes a sample, e.g.
+`cmake --preset dat -DMELEE_DAT_SAMPLES_ALL="PlFx.dat;Gr*.dat"`, and
+`MELEE_DAT_SAMPLES_EXCLUDE` type globs never to sample (data they point to
+stays bytes), for records too bulky to want in C.
+
 Use `samples report` for the verdict: objdiff's own report measures data
 per section and misses relocation differences. To check a type change, edit
 the header and rebuild the preset.