Remove asm-processor in favor of preprocess.sh

Makefile

@@ -14,17 +14,21 @@ ifeq ($(OS),Windows_NT)
   DETECTED_OS = windows
   MAKE = make
   VENV_BIN_DIR = Scripts
+  ICONV = iconv
 else
   UNAME_S := $(shell uname -s)
   ifeq ($(UNAME_S),Linux)
     DETECTED_OS = linux
     MAKE = make
     VENV_BIN_DIR = bin
+    ICONV = iconv
   endif
   ifeq ($(UNAME_S),Darwin)
     DETECTED_OS = macos
     MAKE = gmake
     VENV_BIN_DIR = bin
+    # The default iconv on macOS has some differences from GNU iconv, so we use the Homebrew version instead
+    ICONV = $(shell brew --prefix)/opt/libiconv/bin/iconv
   endif
 endif
 
@@ -126,8 +130,7 @@ CC_CHECK_WARNINGS := -Wall -Wextra -Wno-unknown-pragmas -Wno-unused-parameter -W
 CC_CHECK_WARNINGS += -Werror=implicit-int -Werror=implicit-function-declaration -Werror=int-conversion -Werror=incompatible-pointer-types
 # Have CC_CHECK pretend to be a MIPS compiler
 MIPS_BUILTIN_DEFS := -DMIPSEB -D_MIPS_FPSET=16 -D_MIPS_ISA=2 -D_ABIO32=1 -D_MIPS_SIM=_ABIO32 -D_MIPS_SZINT=32 -D_MIPS_SZPTR=32
-# The -MMD flags additionaly creates a .d file with the same name as the .o file.
-CC_CHECK_FLAGS    := -MMD -MP -fno-builtin -fsyntax-only -funsigned-char -fdiagnostics-color -std=gnu89 -m32 -DNON_MATCHING -DAVOID_UB -DCC_CHECK=1
+CC_CHECK_FLAGS    := -fno-builtin -fsyntax-only -funsigned-char -fdiagnostics-color -std=gnu89 -m32 -DNON_MATCHING -DAVOID_UB -DCC_CHECK=1
 
 ifneq ($(WERROR), 0)
   CC_CHECK_WARNINGS += -Werror
@@ -385,7 +388,6 @@ $(BUILD_DIR)/src/boot/libc64/%.o: OPTFLAGS := -O2
 $(BUILD_DIR)/src/audio/%.o: OPTFLAGS := -O2
 
 $(BUILD_DIR)/assets/%.o: OPTFLAGS := -O1
-$(BUILD_DIR)/assets/%.o: ASM_PROC_FLAGS := 
 
 # file flags
 $(BUILD_DIR)/src/libultra/libc/ll.o: OPTFLAGS := -O1
@@ -408,15 +410,13 @@ $(BUILD_DIR)/src/code/osFlash.o: MIPS_VERSION := -mips1
 # cc & asm-processor
 $(BUILD_DIR)/src/libultra/%.o: CC := $(CC_OLD)
 
-$(BUILD_DIR)/src/boot/%.o: CC := $(ASM_PROC) $(ASM_PROC_FLAGS) $(CC) -- $(AS) $(ASFLAGS) --
-
-$(BUILD_DIR)/src/code/%.o: CC := $(ASM_PROC) $(ASM_PROC_FLAGS) $(CC) -- $(AS) $(ASFLAGS) --
-$(BUILD_DIR)/src/audio/%.o: CC := $(ASM_PROC) $(ASM_PROC_FLAGS) $(CC) -- $(AS) $(ASFLAGS) --
-
-$(BUILD_DIR)/src/overlays/%.o: CC := $(ASM_PROC) $(ASM_PROC_FLAGS) $(CC) -- $(AS) $(ASFLAGS) --
-
-$(BUILD_DIR)/assets/%.o: CC := $(ASM_PROC) $(ASM_PROC_FLAGS) $(CC) -- $(AS) $(ASFLAGS) --
+# For using asm_processor on some files:
+#$(BUILD_DIR)/.../%.o: CC := $(PYTHON) $(ASM_PROC) $(ASM_PROC_FLAGS) $(CC) -- $(AS) $(ASFLAGS) --
 
+ifeq ($(PERMUTER),)  # permuter + preprocess.sh misbehaves, permuter doesn't care about rodata diffs or bss ordering so just don't use it in that case
+# Handle encoding (UTF-8 -> EUC-JP) and custom pragmas
+$(BUILD_DIR)/src/%.o: CC := ./tools/buildtools/preprocess.sh -v $(VERSION) -i $(ICONV) -- $(CC)
+endif
 
 $(SHIFTJIS_O_FILES): CC_CHECK_WARNINGS += -Wno-multichar -Wno-type-limits -Wno-overflow
 
@@ -586,7 +586,7 @@ $(BUILD_DIR)/src/overlays/%_reloc.o: $(BUILD_DIR)/$(SPEC)
 
 # Incremental link z_game_over data into rodata
 $(BUILD_DIR)/src/code/z_game_over.o: src/code/z_game_over.c
-	$(CC_CHECK_COMP) $(CC_CHECK_FLAGS) $(IINC) $(CC_CHECK_WARNINGS) $(C_DEFINES) $(MIPS_BUILTIN_DEFS) -o $(@:.o=.tmp) $<
+	$(CC_CHECK_COMP) $(CC_CHECK_FLAGS) $(IINC) $(CC_CHECK_WARNINGS) $(C_DEFINES) $(MIPS_BUILTIN_DEFS) $<
 	$(CC) -c $(CFLAGS) $(IINC) $(WARNINGS) $(C_DEFINES) $(MIPS_VERSION) $(ENDIAN) $(OPTFLAGS) -o $(@:.o=.tmp) $<
 	$(LD) -r -T linker_scripts/data_with_rodata.ld $(@:.o=.tmp) -o $@
 	@$(RM) $(@:.o=.tmp)
@@ -595,27 +595,27 @@ $(BUILD_DIR)/src/code/z_game_over.o: src/code/z_game_over.c
 
 $(SHIFTJIS_O_FILES): $(BUILD_DIR)/src/%.o: src/%.c
 	$(SHIFTJIS_CONV) -o $(@:.o=.enc.c) $<
-	$(CC_CHECK_COMP) $(CC_CHECK_FLAGS) $(IINC) $(CC_CHECK_WARNINGS) $(C_DEFINES) $(MIPS_BUILTIN_DEFS) -o $@ $(@:.o=.enc.c)
+	$(CC_CHECK_COMP) $(CC_CHECK_FLAGS) $(IINC) $(CC_CHECK_WARNINGS) $(C_DEFINES) $(MIPS_BUILTIN_DEFS) $(@:.o=.enc.c)
 	$(CC) -c $(CFLAGS) $(IINC) $(WARNINGS) $(C_DEFINES) $(MIPS_VERSION) $(ENDIAN) $(OPTFLAGS) -o $@ $(@:.o=.enc.c)
 	$(OBJDUMP_CMD)
 	$(RM_MDEBUG)
 
 $(BUILD_DIR)/src/libultra/libc/ll.o: src/libultra/libc/ll.c
-	$(CC_CHECK_COMP) $(CC_CHECK_FLAGS) $(IINC) $(CC_CHECK_WARNINGS) $(C_DEFINES) $(MIPS_BUILTIN_DEFS) -o $@ $<
+	$(CC_CHECK_COMP) $(CC_CHECK_FLAGS) $(IINC) $(CC_CHECK_WARNINGS) $(C_DEFINES) $(MIPS_BUILTIN_DEFS) $<
 	$(CC) -c $(CFLAGS) $(IINC) $(WARNINGS) $(C_DEFINES) $(MIPS_VERSION) $(ENDIAN) $(OPTFLAGS) -o $@ $<
 	$(PYTHON) tools/set_o32abi_bit.py $@
 	$(OBJDUMP_CMD)
 	$(RM_MDEBUG)
 
 $(BUILD_DIR)/src/libultra/libc/llcvt.o: src/libultra/libc/llcvt.c
-	$(CC_CHECK_COMP) $(CC_CHECK_FLAGS) $(IINC) $(CC_CHECK_WARNINGS) $(C_DEFINES) $(MIPS_BUILTIN_DEFS) -o $@ $<
+	$(CC_CHECK_COMP) $(CC_CHECK_FLAGS) $(IINC) $(CC_CHECK_WARNINGS) $(C_DEFINES) $(MIPS_BUILTIN_DEFS) $<
 	$(CC) -c $(CFLAGS) $(IINC) $(WARNINGS) $(C_DEFINES) $(MIPS_VERSION) $(ENDIAN) $(OPTFLAGS) -o $@ $<
 	$(PYTHON) tools/set_o32abi_bit.py $@
 	$(OBJDUMP_CMD)
 	$(RM_MDEBUG)
 
 $(BUILD_DIR)/%.o: %.c
-	$(CC_CHECK_COMP) $(CC_CHECK_FLAGS) $(IINC) $(CC_CHECK_WARNINGS) $(C_DEFINES) $(MIPS_BUILTIN_DEFS) -o $@ $<
+	$(CC_CHECK_COMP) $(CC_CHECK_FLAGS) $(IINC) $(CC_CHECK_WARNINGS) $(C_DEFINES) $(MIPS_BUILTIN_DEFS) $<
 	$(CC) -c $(CFLAGS) $(IINC) $(WARNINGS) $(C_DEFINES) $(MIPS_VERSION) $(ENDIAN) $(OPTFLAGS) -o $@ $<
 	$(OBJDUMP_CMD)
 	$(RM_MDEBUG)

docs/BUILDING_MACOS.md

@@ -12,12 +12,13 @@ For macOS, use Homebrew to install the following dependencies:
 * libpng
 * bash
 * libxml2
+* libiconv
 
 You can install them with the following commands:
 
 ```bash
 brew update
-brew install coreutils make python3 libpng bash libxml2
+brew install coreutils make python3 libpng bash libxml2 libiconv
 ```
 
 (The repository expects Homebrew-installed programs to be either linked correctly in `$PATH` etc. or in their default locations.)
@@ -72,10 +73,10 @@ make -j
 sudo make install
 ```
 
-Edit your `~/.bash_profile`/`~/.zsh_profile` (or whichever shell you use) to add the new binutils binaries to the system PATH
+Edit your `~/.bash_profile`/`~/.zprofile` (or whichever shell you use) to add the new binutils binaries to the system PATH
 
 ```bash
-echo "export PATH=$PATH:/opt/cross/bin" >> ~/.bash_profile
+echo 'export PATH="$PATH:/opt/cross/bin"' >> ~/.bash_profile
 ```
 
 Reload ~/.bash_profile (or just launch a new terminal tab)

include/z64cutscene_commands.h

@@ -6,7 +6,33 @@
 
 /**
  * Cutscene scripts are arrays of `CutsceneData` words, including bit-packed integers and floats.
- */
+ *
+ * Most command macros have unused arguments. This is to account for the vanilla assets setting specific values
+ * that don't end up being used by any code. They can safely be set to anything, as they aren't used in the
+ * implementation.
+ *
+ * It is believed the original tool used for cutscenes handled most commands the same way, using similar
+ * fields, and the code would have accessed them using common structs. Given this, the unused values observed in vanilla
+ * assets may appear to map to a variable that makes sense, even if it doesn't end up being used in the code. It
+ * probably isn't garbage data.
+ *
+ * This codebase goes with specialized structs and macros to make it easier to follow the code.
+ * Note this common struct design is still partially reflected in all commands having a `startFrame` and `endFrame`,
+ * when sometimes only the `startFrame` matters (as documented).
+ */
+
+/**
+ * CMD_F expects an (IEEE 754) encoded float (colloquially "in hex", such as `0x42280000`),
+ * rather than a C float literal (such as `42.0f`).
+ * Float literals cannot be used because cutscenes are arrays of union type CutsceneData, which may contain integers and floats.
+ * Regardless of CutsceneData having a float member, initializing with a float will cast the float to s32.
+ * Designated initializers (added in C99) would solve this problem but are not supported by IDO (C89 and some extensions).
+ */
+#ifdef __GNUC__
+#define CS_FLOAT(ieee754bin, f) (f)
+#else
+#define CS_FLOAT(ieee754bin, f) (ieee754bin)
+#endif
 
 /**
  * Marks the beginning of a cutscene script.

src/audio/lib/effects.c

@@ -271,8 +271,8 @@ f32 AudioEffects_UpdateAdsr(AdsrState* adsr) {
         case ADSR_STATUS_START_LOOP:
             adsr->envelopeIndex = 0;
             adsr->action.s.status = ADSR_STATUS_LOOP;
+        retry:;
             FALLTHROUGH;
-        retry:
         case ADSR_STATUS_LOOP:
             adsr->delay = adsr->envelope[adsr->envelopeIndex].delay;
             switch (adsr->delay) {

src/boot/CIC6105.c

@@ -1,9 +1,10 @@
-#include "prevent_bss_reordering.h"
 #include "CIC6105.h"
 
 #include "build.h"
 #include "fault.h"
 
+#pragma increment_block_number "n64-us:200"
+
 s32 gCICAddr1Val;
 s32 gCICAddr2Val;
 FaultClient sRomInfoFaultClient;

src/boot/idle.c

@@ -1,5 +1,3 @@
-#include "prevent_bss_reordering.h"
-
 #include "stdbool.h"
 
 #include "buffers.h"
@@ -11,6 +9,8 @@
 #include "z64dma.h"
 #include "z64thread.h"
 
+#pragma increment_block_number "n64-us:128"
+
 // Variables are put before most headers as a hacky way to bypass bss reordering
 struct IrqMgr gIrqMgr;
 STACK(sIrqMgrStack, 0x500);

src/boot/z_std_dma.c

@@ -18,7 +18,6 @@
  * There are some additional provisions to ensure that audio DMA is particularly high-speed, the audio data is assumed
  * to be uncompressed and the request queue and address translation is skipped.
  */
-#include "prevent_bss_reordering.h"
 #include "z64dma.h"
 
 #include "carthandle.h"

src/buffers/gfxbuffers.c

@@ -1,4 +1,3 @@
-#include "prevent_bss_reordering.h"
 #include "buffers.h"
 
 u8 gGfxSPTaskYieldBuffer[OS_YIELD_DATA_SIZE] ALIGNED(16);

src/code/main.c

@@ -1,7 +1,8 @@
-#include "prevent_bss_reordering.h"
 #include "ultra64.h"
 #include "stack.h"
 
+#pragma increment_block_number "n64-us:128"
+
 // Variables are put before most headers as a hacky way to bypass bss reordering
 OSMesgQueue sSerialEventQueue;
 OSMesg sSerialMsgBuf[1];