From 1fa17953c46f99102e7dccb39c168be71b47bbad Mon Sep 17 00:00:00 2001 From: Rob Realmuto Date: Tue, 19 Sep 2023 17:34:28 -0400 Subject: [PATCH 01/10] Add MMR custom music support and OOTRS improvements --- Audiobank.py | 185 +++++++++++++++ Music.py | 435 ++++++++++++++++++---------------- MusicHelpers.py | 209 ++++++++++++++++ Sequence.py | 42 ++++ Unittest.py | 150 +++++++++++- data/Music/update_ootrs_v2.py | 308 ++++++++++++++++++++++++ 6 files changed, 1128 insertions(+), 201 deletions(-) create mode 100644 Audiobank.py create mode 100644 MusicHelpers.py create mode 100644 Sequence.py create mode 100644 data/Music/update_ootrs_v2.py diff --git a/Audiobank.py b/Audiobank.py new file mode 100644 index 000000000..9a7bc5745 --- /dev/null +++ b/Audiobank.py @@ -0,0 +1,185 @@ +from io import FileIO +from Rom import Rom + +# Container for storing Audiotable, Audiobank, Audiotable_index, Audiobank_index +class Audiobin: + def __init__(self, _Audiobank: bytearray, _Audiobank_index: bytearray, _Audiotable: bytearray, _Audiotable_index: bytearray): + self.Audiobank: bytearray = _Audiobank + self.Audiobank_index: bytearray = _Audiobank_index + self.Audiotable: bytearray = _Audiotable + self.Audiotable_index: bytearray = _Audiotable_index + + num_banks = int.from_bytes(self.Audiobank_index[0:2], 'big') + self.audiobanks: list[AudioBank] = [] + for i in range(0, num_banks): + index = 0x10 + (0x10*i) + curr_entry = self.Audiobank_index[index:index+0x10] + audiobank: AudioBank = AudioBank(curr_entry, self.Audiobank, self.Audiotable, self.Audiotable_index) + self.audiobanks.append(audiobank) + + def find_sample_in_audiobanks(self, sample_data: bytearray): + for audiobank in self.audiobanks: + for drum in audiobank.drums: + if drum and drum.sample: + if drum.sample.data == sample_data: + return drum.sample + for instrument in audiobank.instruments: + if instrument: + if instrument.highNoteSample and instrument.highNoteSample.data == sample_data: + return instrument.highNoteSample + if instrument.lowNoteSample and instrument.lowNoteSample.data == sample_data: + return instrument.lowNoteSample + if instrument.normalNoteSample and instrument.normalNoteSample.data == sample_data: + return instrument.normalNoteSample + for sfx in audiobank.SFX: + if sfx and sfx.sample: + if sfx.sample.data == sample_data: + return sfx.sample + return None + +class Sample: + def __init__(self, bankdata: bytearray, audiotable_file: bytearray, audiotable_index: bytearray, sample_offset: int, audiotable_id: int, parent): + # Process the sample + self.parent = parent + self.bank_offset = sample_offset + self.sample_header = bankdata[sample_offset:sample_offset + 0x10] + self.codec = (self.sample_header[0] & 0xF0) >> 4 + self.medium = (self.sample_header[0] & 0x0C) >> 2 + self.size = int.from_bytes(self.sample_header[1:4], 'big') + self.addr = int.from_bytes(self.sample_header[4:8], 'big') + + if audiotable_file and self.addr > len(audiotable_file): # The offset is higher than the total size of audiotable so we'll assume it doesn't actually exist. # We'll need to get the sample data from ZSOUND files in the archive. + self.data = None + self.addr = -1 + return + # Read the audiotable pointer table entry + if audiotable_file and audiotable_index: + audiotable_index_offset = 0x10 + (audiotable_id * 0x10) + audiotable_entry = audiotable_index[audiotable_index_offset:audiotable_index_offset + 0x10] + audiotable_offset = int.from_bytes(audiotable_entry[0:4], 'big') + sample_address = audiotable_offset + self.addr + self.audiotable_addr = sample_address + # Read the sample data + self.data = audiotable_file[sample_address:sample_address+self.size] + else: + self.audiotable_addr = -1 + self.data = None + +# Loads an audiobank and it's corresponding instrument/drum/sfxs +class AudioBank: + + # Constructor: + # table_entry - 0x10 byte audiobank entry which contains info like the bank offset, size, number of instruments, etc. + # audiobank_file - the Audiobank file as a byte array + # audiotable_file - the Audiotable file as a byte array + # audiotable_index - the Audiotable index (pointer table) which provides an offsets into the Audiotable file where a bank's instrument samples offsets are calculated from. + def __init__(self, table_entry: bytearray, audiobank_file: bytearray, audiotable_file: bytearray, audiotable_index: bytearray) -> None: + + # Process bank entry + self.bank_offset: int = int.from_bytes(table_entry[0:4], 'big') # Offset of the bank in the Audiobank file + self.size: int = int.from_bytes(table_entry[4:8], 'big') # Size of the bank, in bytes + self.load_location: int = table_entry[8] # ROM/RAM/DISK + self.type: int = table_entry[9] + self.audiotable_id: int = table_entry[10] # Read audiotable id from the table entry. Instrument data offsets are in relation to this + self.unk: int = table_entry[11] # 0xFF + self.num_instruments: int = table_entry[12] + self.num_drums: int = table_entry[13] + self.num_sfx: int = int.from_bytes(table_entry[14:16], 'big') + self.bank_data = audiobank_file[self.bank_offset:self.bank_offset + self.size] + self.original_data = self.bank_data.copy() + self.table_entry: bytearray = table_entry + self.duplicate_banks: list[AudioBank] = [] + # Process the bank + + # Read drums + self.drums: list[Drum] = [] + drum_offset = int.from_bytes(self.bank_data[0:4], 'big') # Get the drum pointer. This is the first uint32 in the bank. Points to a list of drum offsets of length num_drums + for i in range(0, self.num_drums): # Read each drum + offset = drum_offset + 4*i + offset = int.from_bytes(self.bank_data[offset:offset+4], 'big') + drum = Drum(i, self.bank_data, audiotable_file, audiotable_index, offset, self.audiotable_id) if offset != 0 else None + self.drums.append(drum) + + # Read SFX + self.SFX: list[SFX] = [] + sfx_offset = int.from_bytes(self.bank_data[4:8], 'big') # Get the SFX pointer. this is the second uint32 in the bank. Points to a list of Sound objects which are 8 bytes each (Sample offsets + tuning) + for i in range(0, self.num_sfx): # Read each SFX + offset = sfx_offset + 8*i + sfx = SFX(i, self.bank_data, audiotable_file, audiotable_index, offset, self.audiotable_id) if offset != 0 else None + self.SFX.append(sfx) + + self.instruments: list[Instrument] = [] + # Read the instruments + for i in range(0, self.num_instruments): + offset = 0x08 + 4*i + instr_offset = int.from_bytes(self.bank_data[offset:offset+4], 'big') + instrument: Instrument = Instrument(i, self.bank_data, audiotable_file, audiotable_index, instr_offset, self.audiotable_id) if instr_offset != 0 else None + self.instruments.append(instrument) + + def __str__(self): + return "Offset: " + hex(self.bank_offset) + ", " + "Len:" + hex(self.size) + + def get_all_samples(self) -> list[Sample]: + all_sounds = self.drums + self.instruments + self.SFX + all_samples: list[Sample] = [] + for sound in all_sounds: + if type(sound) == Instrument: + instrument: Instrument = sound + if instrument.highNoteSample: + all_samples.append(instrument.highNoteSample) + if instrument.lowNoteSample: + all_samples.append(instrument.lowNoteSample) + if instrument.normalNoteSample: + all_samples.append(instrument.normalNoteSample) + + elif type(sound) == Drum: + drum: Drum = sound + if drum.sample: + all_samples.append(drum.sample) + elif type(sound) == SFX: + sfx: SFX = sound + if sfx.sample: + all_samples.append(sfx.sample) + return all_samples + + def build_entry(self, offset: int) -> bytes: + bank_entry: bytes = offset.to_bytes(4, 'big') + bank_entry += len(self.bank_data).to_bytes(4, 'big') + bank_entry += self.table_entry[8:16] + return bank_entry + +class Drum: + def __init__(self, drum_id: int, bankdata: bytearray, audiotable_file: bytearray, audiotable_index: bytearray, drum_offset: int, audiotable_id: int) -> None: + self.drum_id = drum_id + self.releaseRate = bankdata[drum_offset] + self.pan = bankdata[drum_offset + 1] + self.sampleOffset = int.from_bytes(bankdata[drum_offset + 4:drum_offset+8], 'big') + self.sampleTuning = int.from_bytes(bankdata[drum_offset + 8:drum_offset+12], 'big') + self.envelopePointOffset = int.from_bytes(bankdata[drum_offset+12:drum_offset+16], 'big') + self.sample: Sample = Sample(bankdata, audiotable_file, audiotable_index, self.sampleOffset, audiotable_id, self) + +class SFX: + def __init__(self, sfx_id: int, bankdata: bytearray, audiotable_file: bytearray, audiotable_index: bytearray, sfx_offset: int, audiotable_id: int) -> None: + self.sfx_id = sfx_id + self.sampleOffset = int.from_bytes(bankdata[sfx_offset:sfx_offset+4], 'big') + self.sampleTuning = int.from_bytes(bankdata[sfx_offset+4:sfx_offset+8], 'big') + self.sample: Sample = Sample(bankdata, audiotable_file, audiotable_index, self.sampleOffset, audiotable_id, self) + + +class Instrument: + def __init__(self, inst_id: int, bankdata: bytearray, audiotable_file: bytearray, audiotable_index: bytearray, instr_offset: int, audiotable_id: int) -> None: + self.inst_id = inst_id + self.normalRangeLo = bankdata[instr_offset + 1] + self.normalRangeHi = bankdata[instr_offset + 2] + self.releaseRate = bankdata[instr_offset + 3] + self.AdsrEnvelopePointOffset = int.from_bytes(bankdata[instr_offset + 4:instr_offset+8], 'big') + self.lowNoteSampleOffset = int.from_bytes(bankdata[instr_offset + 8:instr_offset+12], 'big') + self.lowNoteTuning = int.from_bytes(bankdata[instr_offset + 12: instr_offset + 16], 'big') + self.normalNoteSampleOffset = int.from_bytes(bankdata[instr_offset + 16:instr_offset+20], 'big') + self.normalNoteTuning = int.from_bytes(bankdata[instr_offset + 20:instr_offset + 24], 'big') + self.highNoteSampleOffset = int.from_bytes(bankdata[instr_offset + 24:instr_offset+28], 'big') + self.highNoteSampleTuning = int.from_bytes(bankdata[instr_offset + 28:instr_offset+32], 'big') + self.lowNoteSample: Sample = Sample(bankdata, audiotable_file, audiotable_index, self.lowNoteSampleOffset, audiotable_id, self) if self.lowNoteSampleOffset != 0 else None + self.normalNoteSample: Sample = Sample(bankdata, audiotable_file, audiotable_index, self.normalNoteSampleOffset, audiotable_id, self) if self.normalNoteSampleOffset != 0 else None + self.highNoteSample: Sample = Sample(bankdata, audiotable_file, audiotable_index, self.highNoteSampleOffset, audiotable_id, self) if self.highNoteSampleOffset != 0 else None + diff --git a/Music.py b/Music.py index 74bb47b3c..5d8407b93 100644 --- a/Music.py +++ b/Music.py @@ -1,5 +1,6 @@ # Much of this is heavily inspired from and/or based on az64's / Deathbasket's MM randomizer from __future__ import annotations +from enum import Enum import io import itertools import os @@ -7,8 +8,11 @@ import zipfile from collections.abc import Iterable from typing import TYPE_CHECKING, Optional, Any +from Audiobank import * +from MusicHelpers import process_sequence_mmr_zseq, process_sequence_mmrs, process_sequence_ootrs from Rom import Rom +from Sequence import Sequence, SequenceGame from Utils import compare_version, data_path if TYPE_CHECKING: @@ -117,60 +121,6 @@ ("File Select", 0x57), ) - -class Bank: - def __init__(self, index: int, meta: bytearray, data: bytes) -> None: - self.index: int = index - self.meta: bytearray = meta - self.data: bytes = data - self.zsounds: dict[int, dict[str, Any]] = {} - - def add_zsound(self, tempaddr: int, zsound: dict[str, Any]) -> None: - self.zsounds[tempaddr] = zsound - - def get_entry(self, offset: int) -> bytes: - bank_entry = offset.to_bytes(4, 'big') - bank_entry += len(self.data).to_bytes(4, 'big') - bank_entry += self.meta - return bank_entry - - def update_zsound_pointers(self) -> None: - for zsound_tempaddr in self.zsounds.keys(): - self.data = self.data.replace(zsound_tempaddr.to_bytes(4, byteorder='big'), self.zsounds[zsound_tempaddr]['offset'].to_bytes(4, byteorder='big')) - - -# Represents the information associated with a sequence, aside from the sequence data itself -class Sequence: - def __init__(self, name: str, cosmetic_name: str, type: int = 0x0202, instrument_set: int | str = 0x03, - replaces: int = -1, vanilla_id: int = -1, seq_file: Optional[str] = None, new_instrument_set: bool = False, - zsounds: Optional[list[dict[str, str]]] = None) -> None: - self.name: str = name - self.seq_file = seq_file - self.cosmetic_name: str = cosmetic_name - self.replaces: int = replaces - self.vanilla_id: int = vanilla_id - self.type: int = type - self.new_instrument_set: bool = new_instrument_set - self.zsounds: Optional[list[dict[str, str]]] = zsounds - self.zbank_file: Optional[str] = None - self.bankmeta: Optional[str] = None - - self.instrument_set: int = 0x0 - if isinstance(instrument_set, str): - if instrument_set == '-': - self.new_instrument_set = True - else: - self.instrument_set = int(instrument_set, 16) - else: - self.instrument_set = instrument_set - - def copy(self) -> Sequence: - copy = Sequence(self.name, self.cosmetic_name, self.type, self.instrument_set, self.replaces, self.vanilla_id, self.seq_file, self.new_instrument_set, self.zsounds) - copy.zbank_file = self.zbank_file - copy.bankmeta = self.bankmeta - return copy - - # Represents actual sequence data, along with metadata for the sequence data block class SequenceData: def __init__(self) -> None: @@ -199,8 +149,8 @@ def process_sequences(rom: Rom, ids: Iterable[tuple[str, int]], seq_type: str = id = bgm[1] # Create new sequences - seq = Sequence(name, cosmetic_name, type, instrument_set, vanilla_id = id) - target = Sequence(name, cosmetic_name, type, instrument_set, replaces = id) + seq = Sequence(name, cosmetic_name, seq_type, type, instrument_set, vanilla_id = id) + target = Sequence(name, cosmetic_name, seq_type, type, instrument_set, replaces = id) # Special handling for file select/fairy fountain if seq.vanilla_id != 0x57 and cosmetic_name not in disabled_source_sequences: @@ -233,89 +183,17 @@ def process_sequences(rom: Rom, ids: Iterable[tuple[str, int]], seq_type: str = if fname in seq_exclusion_list: continue - # Find .ootrs zip files - if fname.endswith('.ootrs'): - skip = False - # Open zip file - filepath = os.path.join(dirpath, fname) - with zipfile.ZipFile(filepath) as zip: - # Make sure meta file and seq file exists - meta_file = None - seq_file = None - zbank_file = None - bankmeta_file = None - for f in zip.namelist(): - if f.endswith(".meta"): - meta_file = f - continue - if f.endswith(".seq"): - seq_file = f - continue - if f.endswith(".zbank"): - if not include_custom_audiobanks: # Check if we are excluding sequences with custom banks - skip = True - break - zbank_file = f - continue - if f.endswith(".bankmeta"): - bankmeta_file = f - continue - - if skip: - continue - - if not meta_file: - raise FileNotFoundError(f'No .meta file in: "{fname}". This should never happen') - if not seq_file: - raise FileNotFoundError(f'No .seq file in: "{fname}". This should never happen') - if zbank_file and not bankmeta_file: - raise FileNotFoundError(f'Custom track "{fname}" contains .zbank but no .bankmeta') - - # Read meta info - try: - with zip.open(meta_file, 'r') as stream: - lines = io.TextIOWrapper(stream).readlines() # Use TextIOWrapper in order to get text instead of binary from the seq. - # Strip newline(s) - lines = [line.rstrip() for line in lines] - except Exception as ex: - raise FileNotFoundError(f'Error reading meta file for: "{fname}". This should never happen') - - # Create new sequence, checking third line for correct type - if (len(lines) > 2 and (lines[2].lower() == seq_type.lower() or lines[2] == '')) or (len(lines) <= 2 and seq_type == 'bgm'): - seq = Sequence(filepath, lines[0], seq_file = seq_file, instrument_set = lines[1]) - if zbank_file: - seq.zbank_file = zbank_file - seq.bankmeta = bankmeta_file - seq.zsounds = [] - if seq.instrument_set < 0x00 or seq.instrument_set > 0x25: - raise Exception(f'{seq.name}: Sequence instrument must be in range [0x00, 0x25]') - if seq.cosmetic_name == "None": - raise Exception(f'{seq.name}: Sequences should not be named "None" as that is used for disabled music.') - if seq.cosmetic_name in sequences: - raise Exception(f'{seq.name} Sequence names should be unique. Duplicate sequence name: {seq.cosmetic_name}') - - if seq.cosmetic_name not in disabled_source_sequences: - sequences[seq.cosmetic_name] = seq - - if len(lines) >= 4: - seq_groups = lines[3].split(',') - for group in seq_groups: - group = group.strip() - if group not in groups: - groups[group] = [] - groups[group].append(seq.cosmetic_name) - - # Process ZSOUND lines. Make these lines in the format of ZSOUND:file_path:temp_addr - for line in lines: - tokens = line.split(":") - if tokens[0] == "ZSOUND": - zsound_file = tokens[1] - zsound_tempaddr = tokens[2] - zsound = { - "file": tokens[1], - "tempaddr": tokens[2] - } - seq.zsounds.append(zsound) + filepath = os.path.join(dirpath, fname) + seq = None + if fname.lower().endswith('.ootrs'): + seq = process_sequence_ootrs(filepath, fname, seq_type, include_custom_audiobanks, groups) + elif fname.lower().endswith('.zseq'): + seq = process_sequence_mmr_zseq(filepath, fname, seq_type, include_custom_audiobanks, groups) + elif fname.lower().endswith('.mmrs'): + seq = process_sequence_mmrs(filepath, fname, seq_type, include_custom_audiobanks, groups) + if seq: + sequences[seq.cosmetic_name] = seq + continue return sequences, target_sequences, groups @@ -394,7 +272,7 @@ def rebuild_sequences(rom: Rom, sequences: list[Sequence], log: CosmeticsLog, sy old_sequences.append(entry) # List of sequences containing the new sequence data - new_sequences = [] + new_sequences: list[SequenceData] = [] address = 0 # Byte array to hold the data for the whole audio sequence new_audio_sequence = [] @@ -417,10 +295,19 @@ def rebuild_sequences(rom: Rom, sequences: list[Sequence], log: CosmeticsLog, sy else: # Read sequence info try: - with zipfile.ZipFile(seq.name) as zip: - with zip.open(seq.seq_file, 'r') as stream: + if seq.name.endswith('.zseq'): + with open(seq.name, 'rb') as stream: new_entry.data = bytearray(stream.read()) new_entry.size = len(new_entry.data) + else: + with zipfile.ZipFile(seq.name) as zip: + with zip.open(seq.seq_file, 'r') as stream: + new_entry.data = bytearray(stream.read()) + new_entry.size = len(new_entry.data) + # Align sequences to 0x10 + if new_entry.size % 0x10 != 0: + new_entry.data.extend(bytearray(0x10 - (new_entry.size % 0x10))) + new_entry.size += 0x10 - (new_entry.size % 0x10) if new_entry.size <= 0x10: raise Exception(f'Invalid sequence file "{seq.name}.seq"') new_entry.data[1] = 0x20 @@ -430,14 +317,17 @@ def rebuild_sequences(rom: Rom, sequences: list[Sequence], log: CosmeticsLog, sy new_entry.size = old_sequences[i].size new_entry.data = old_sequences[i].data + # Check if we have already added this sequence. This happens if we have less available sequences than total sequences + for new_seq in new_sequences: + if (new_entry.size == new_seq.size) and (new_entry.data == new_seq.data): + new_entry.address = new_seq.address # Use the address of the sequence that has already been added + new_entry.data = [] # Clear the data so it doesn't get added again. + break + new_sequences.append(new_entry) # Concatenate the full audio sequence and the new sequence data if new_entry.data != [] and new_entry.size > 0: - # Align sequences to 0x10 - if new_entry.size % 0x10 != 0: - new_entry.data.extend(bytearray(0x10 - (new_entry.size % 0x10))) - new_entry.size += 0x10 - (new_entry.size % 0x10) new_audio_sequence.extend(new_entry.data) # Increment the current address by the size of the new sequence address += new_entry.size @@ -510,8 +400,8 @@ def rebuild_sequences(rom: Rom, sequences: list[Sequence], log: CosmeticsLog, sy rom.write_bytes(bank_index_base + 0x270 + 0x10 * i, bank_entry) # Write the new entry at the end of the bank table. rom.write_byte(bank_index_base + 0x01, 0x4C) # Updates AudioBank Index Header if no custom banks are present as this would be 0x26 which would crash the game if a fanfare was played - added_banks = [] # Store copies of all the banks we've added - added_instruments = [] # Store copies of all the instruments we've added + added_banks: list[AudioBank] = [] # Store copies of all the banks we've added + added_samples: list[Sample] = [] # Store copies of all the samples we've added new_bank_index = 0x4C instr_data = bytearray(0) # Store all the new instrument data that will be added to the end of audiotable @@ -522,58 +412,198 @@ def rebuild_sequences(rom: Rom, sequences: list[Sequence], log: CosmeticsLog, sy instr_offset_in_file = audiotable_size bank_table_base = 0 + + # Load OOT Audiobin + AUDIOBANK_INDEX_ADDR = 0x00B896A0 + AUDIOBANK_ADDR = 0xD390 + AUDIOTABLE_INDEX_ADDR = 0xB8A1C0 + AUDIOTABLE_ADDR = 0x79470 + + audiobank_index = rom.read_bytes(AUDIOBANK_INDEX_ADDR, 0x2A0) + audiobank = rom.read_bytes(AUDIOBANK_ADDR, 0x1CA50) + audiotable_index = rom.read_bytes(AUDIOTABLE_INDEX_ADDR, 0x80) # Read audiotable index into bytearray + audiotable = rom.read_bytes(AUDIOTABLE_ADDR, 0x460AD0) # Read audiotable (samples) into bytearray + + oot_audiobin: Audiobin = Audiobin(audiobank, audiobank_index, audiotable, audiotable_index) + + # Load MM Audiobin if needed + mm_audiobin: Audiobin = None + for i in range(0x6E): # Loop through all the replacement sequences + j = replacement_dict.get(i if new_sequences[i].size else new_sequences[i].address, None) + if j and j.game == SequenceGame.MM: # we have at least one MM sequence so load the audiobin + with zipfile.ZipFile(os.path.join(data_path(), 'Music', 'MM.audiobin')) as mm_audiobin_zip: + mm_audiobank = bytearray(mm_audiobin_zip.read("Audiobank")) + mm_audiobank_index = bytearray(mm_audiobin_zip.read("Audiobank_index")) + mm_audiotable = bytearray(mm_audiobin_zip.read("Audiotable")) + mm_audiotable_index = bytearray(mm_audiobin_zip.read("Audiotable_index")) + mm_audiobin = Audiobin(mm_audiobank, mm_audiobank_index, mm_audiotable, mm_audiotable_index) + break + for i in range(0x6E): bank_table_base = (rom.read_int32(symbols['CFG_AUDIOBANK_TABLE_EXTENDED_ADDR']) - 0x80400000) + 0x3480000 seq_bank_base = 0xB89911 + 0xDD + (i * 2) j = replacement_dict.get(i if new_sequences[i].size else new_sequences[i].address, None) if j is not None and j.new_instrument_set: - # Open the .ootrs file - with zipfile.ZipFile(j.name) as zip: - - # Load the .zbank file - with zip.open(j.zbank_file, 'r') as stream: - bankdata = stream.read() - bank = None - - # Check if we have already added this bank - for added_bank in added_banks: - if added_bank.data == bankdata: - bank = added_bank - - if not bank: - bank_meta = bytearray(zip.open(j.bankmeta, 'r').read()) - bank = Bank(new_bank_index, bank_meta, bankdata) - - # Handle any new instruments - for zsound in j.zsounds: - instrument = None - tempaddr = int(zsound["tempaddr"], 16) - curr_instrument_data = zip.open(zsound["file"], 'r').read() - already_added = False - for added_instrument in added_instruments: - if added_instrument['data'] == curr_instrument_data: - # Already added this instrument. Just add it to the bank - instrument = added_instrument - bank.add_zsound(tempaddr, instrument) - already_added = True - if not already_added: - instrument = {'offset': instr_offset_in_file, 'data': curr_instrument_data, - 'size': len(curr_instrument_data), 'name': zsound["file"]} - instr_data += curr_instrument_data - - # Align instrument data to 0x10 - if len(instr_data) % 0x10 != 0: - padding_length = 0x10 - (len(instr_data) % 0x10) - instr_data += (bytearray(padding_length)) - instrument['size'] += padding_length - bank.add_zsound(tempaddr, instrument) - added_instruments.append(instrument) - instr_offset_in_file += instrument['size'] - added_banks.append(bank) - new_bank_index += 1 - - # Update the sequence's bank (instrument set) - rom.write_byte(seq_bank_base, bank.index) + # Open the .ootrs/.mmrs file + newbank: AudioBank = None + vanilla_mm_bank: AudioBank = None + bankdata = None + bank_entry = None + audiobin = oot_audiobin if j.game == SequenceGame.OOT else mm_audiobin # Load the correct audiobin + + # Handle MMR .zseq files. These use vanilla MM banks so just build the bank using the data from the MM audiobin. We'll update it with OOT data later + if j.name.lower().endswith('.zseq'): + # Get the entry from Audiobank_index + offset = 0x10 + j.instrument_set*0x10 + bank_entry = audiobin.Audiobank_index[offset:offset+0x10] + # Get the bank + vanilla_mm_bank = AudioBank(bank_entry, audiobin.Audiobank, audiobin.Audiotable, audiobin.Audiotable_index) + vanilla_mm_bank.table_entry[0x0A] = 1 + bankdata = vanilla_mm_bank.bank_data + else: # MMRS or OOTRs with custom banks + with zipfile.ZipFile(j.name) as zip: + # Check if we have a zbank file because MMR sequences might not. + if j.zbank_file: + bankdata = bytearray(zip.read(j.zbank_file)) # Load the .zbank file + bank_meta = bytearray(zip.open(j.bankmeta, 'r').read()) + bank_entry = bytearray(4) + len(bankdata).to_bytes(4, 'big') + bank_meta + elif j.game == SequenceGame.MM: # MM sequence without a zbank: + offset = 0x10 + j.instrument_set*0x10 + bank_entry = audiobin.Audiobank_index[offset:offset+0x10] + # Get the bank + vanilla_mm_bank = AudioBank(bank_entry, audiobin.Audiobank, audiobin.Audiotable, audiobin.Audiotable_index) + vanilla_mm_bank.table_entry[0x0A] = 1 + bankdata = vanilla_mm_bank.bank_data + else: # Probably should never get here + raise Exception("No bank data available for custom music: " + j.cosmetic_name) + + # Update the bank's cache type based on bgm/fanfare + bank_entry[9] = 1 if j.seq_type == 'fanfare' else 2 + # Check if we have already added this bank + for added_bank in added_banks: + if added_bank.original_data == bankdata: + newbank = added_bank + if added_bank.table_entry[8:16] != bank_entry[8:16]: # We've already added this bank but it has different metadata + found: bool = False + for bank in added_bank.duplicate_banks: + if bank.table_entry[8:16] == bank_entry[8:16]: + found = True + newbank = bank + break + if not found: + dupe_bank = AudioBank(bank_entry, bytearray(0), None, None) + dupe_bank.bank_index = new_bank_index + new_bank_index += 1 + newbank.duplicate_banks.append(dupe_bank) # Make a new bank with just the meta and add it as a duplicate so it can be added separately but point to the same data. + newbank = dupe_bank + break + + if not newbank: + if vanilla_mm_bank: + newbank = vanilla_mm_bank + else: + newbank = AudioBank(bank_entry, bankdata, audiobin.Audiotable, audiobin.Audiotable_index) + newbank.bank_index = new_bank_index + + zsound_samples: list[Sample] = [] + # Handle new zsounds + tempbank = AudioBank(bank_entry, bankdata, None, None) + if j.name.lower().endswith('.ootrs') or j.name.lower().endswith('.mmrs'): + with zipfile.ZipFile(j.name) as zip: + for zsound in j.zsounds: + if zsound['tempaddr']: # Old style/MMR zsound that uses the stupid temp address thing + for sample in tempbank.get_all_samples(): + if sample.addr == zsound['tempaddr']: + parent = sample.parent + if type(parent) == Instrument: + if parent.highNoteSample and parent.highNoteSample.addr == sample.addr: + sample = newbank.instruments[parent.inst_id].highNoteSample + elif parent.normalNoteSample and parent.normalNoteSample.addr == sample.addr: + sample = newbank.instruments[parent.inst_id].normalNoteSample + elif parent.lowNoteSample and parent.lowNoteSample.addr == sample.addr: + sample = newbank.instruments[parent.inst_id].lowNoteSample + if type(parent) == Drum: + sample = newbank.SFX[parent.drum_id] + if type(parent) == SFX: + sample = newbank.SFX[parent.sfx_id] + sample.data = zip.read(zsound['file']) + sample.addr = -1 # Set the sample address to -1 so that we know it's from a zsound + zsound_samples.append(sample) + break + pass + else: + curr_sample_data = zip.read(zsound['file']) + already_added = False + sample: Sample = None + + # Get the sample reference from the new bank + if zsound['type'] == 'DRUM': + sample = newbank.drums[zsound['index']].sample + if zsound['type'] == 'SFX': + sample = newbank.SFX[zsound['index']].sample + if zsound['type'] == 'INST': + if zsound['alt'] == 'LOW': + sample = newbank.instruments[zsound['index']].lowNoteSample + elif zsound['alt'] == 'NORM': + sample = newbank.instruments[zsound['index']].normalNoteSample + elif zsound['alt'] == 'HIGH': + sample = newbank.instruments[zsound['index']].highNoteSample + sample.data = curr_sample_data + sample.addr = -1 # Set the sample address to -1 so that we know it's from a zsound + zsound_samples.append(sample) + + # For MM, update the bank's samples with OOT addresses if they exist: + mm_samples_to_add: list[Sample] = [] + # Cross reference MM instrument data against OOT and update accordingly + if j.game == SequenceGame.MM: + all_samples = newbank.get_all_samples() + all_samples = [sample for sample in all_samples if sample.addr != -1] # Skip samples that are new ZSOUNDS because we'll handle them later + for sample in all_samples: + match = oot_audiobin.find_sample_in_audiobanks(sample.data) + if match: # Found a matching sample in OOT. Just update the bank with the corresponding sample address + # Update the sample's offset in the new bank + newbank.bank_data[sample.bank_offset+4:sample.bank_offset+8] = match.audiotable_addr.to_bytes(4, 'big') + else: # Didn't find a matching sample, Will have to add it later + mm_samples_to_add.append(sample) + i = 0 + while i < len(zsound_samples): + match = oot_audiobin.find_sample_in_audiobanks(zsound_samples[i].data) + if match: # Found a matching sample in OOT. Just update the bank with the corresponding sample address and remove if from zsound list + # Update the sample's offset in the new bank + newbank.bank_data[zsound_samples[i].bank_offset+4:zsound_samples[i].bank_offset+8] = match.audiotable_addr.to_bytes(4, 'big') + zsound_samples.pop(i) + continue + i += 1 + + # Create a list of all samples that need to be added. + # This includes new samples, and samples from MM that don't exist in OOT + for sample_to_add in zsound_samples + mm_samples_to_add: + # Check if we've already added this sample's data + already_added = False + for added_sample in added_samples: + if sample_to_add.data == added_sample.data: + # Already added this sample, just update the bank + newbank.bank_data[sample_to_add.bank_offset+4:sample_to_add.bank_offset+8] = added_sample.audiotable_addr.to_bytes(4, 'big') + already_added = True + break + if not already_added: + instr_data += sample_to_add.data + # Pad instrument data to 0x10 + sample_to_add.audiotable_addr = instr_offset_in_file + instr_offset_in_file += len(sample_to_add.data) + if len(instr_data) % 0x10 != 0: + padding_length = 0x10 - (len(instr_data) % 0x10) + instr_data += (bytearray(padding_length)) + instr_offset_in_file += padding_length + # Update the sample address in the bank data + newbank.bank_data[sample_to_add.bank_offset+4:sample_to_add.bank_offset+8] = sample_to_add.audiotable_addr.to_bytes(4, 'big') + added_samples.append(sample_to_add) + + added_banks.append(newbank) + new_bank_index += 1 + + # Update the sequence's bank (instrument set) + rom.write_byte(seq_bank_base, newbank.bank_index) # Patch the new instrument data into the ROM in a new file. @@ -599,13 +629,18 @@ def rebuild_sequences(rom: Rom, sequences: list[Sequence], log: CosmeticsLog, sy audiobank_data = rom.read_bytes(audiobank_start, audiobank_size) new_bank_offset = len(audiobank_data) for bank in added_banks: - bank.update_zsound_pointers() + # Sample pointers should already be correct now + # bank.update_zsound_pointers() bank.offset = new_bank_offset #absolute_offset = new_audio_banks_addr + new_bank_offset - bank_entry = bank.get_entry(new_bank_offset) - rom.write_bytes(bank_table_base + 0x10 + bank.index * 0x10, bank_entry) - new_bank_data += bank.data - new_bank_offset += len(bank.data) + bank_entry = bank.build_entry(new_bank_offset) + rom.write_bytes(bank_table_base + 0x10 + bank.bank_index * 0x10, bank_entry) + for dupe_bank in bank.duplicate_banks: + bank_entry = bytearray(dupe_bank.build_entry(new_bank_offset)) + bank_entry[4:8] = bank.size.to_bytes(4, 'big') + rom.write_bytes(bank_table_base + 0x10 + dupe_bank.bank_index * 0x10, bank_entry) + new_bank_data += bank.bank_data + new_bank_offset += len(bank.bank_data) # If there is any audiobank data to add, move the entire audiobank file to a new place in ROM. Update the existing dmadata record if len(new_bank_data) > 0: diff --git a/MusicHelpers.py b/MusicHelpers.py new file mode 100644 index 000000000..ac092badf --- /dev/null +++ b/MusicHelpers.py @@ -0,0 +1,209 @@ +# Sequence Processing Functions in the form: +# def process_sequence_func(file_name: str, seq_type: str, include_custom_audiobanks: bool) +# return a Sequence object if it matches seq_type + include_custom_audiobank rule, return None otherwise + +import io +import os +import zipfile +from Sequence import Sequence, SequenceGame +from Utils import data_path + +def process_sequence_mmr_zseq(filepath: str, file_name: str, seq_type: str, include_custom_audiobanks: bool, groups) -> Sequence: + split = file_name.split('.zseq') + base = split[0] + split = base.split('_') + + if len(split) != 3: + raise Exception("Error while processing MMR .zseq " + file_name + ": Not enough parameters in the file name") + + cosmetic_name = split[0] + instrument_set = split[1] + categories = split[2] + + type_match = False + mmrs_categories = categories.split('-') + if seq_type.lower() == 'fanfare' and ('8' in mmrs_categories or '9' in mmrs_categories or '10' in mmrs_categories): + type_match = True + elif seq_type.lower() == 'bgm' and not ('8' in mmrs_categories or '9' in mmrs_categories or '10' in mmrs_categories): + type_match = True + + if not type_match: + return None + + seq_file = filepath + seq = Sequence(filepath, cosmetic_name, seq_type=seq_type, seq_file = seq_file, instrument_set = instrument_set) + seq.game = SequenceGame.MM + seq.new_instrument_set = True + return seq + +def process_sequence_mmrs(filepath: str, file_name: str, seq_type: str, include_custom_audiobanks: bool, groups) -> Sequence: + if not include_custom_audiobanks: + return None + + with zipfile.ZipFile(filepath) as zip: + seq_file = None + zbank_file = None + bankmeta_file = None + for f in zip.namelist(): + # Uncomment if MMR ever decides to wisen up and use a common format + #if f.endswith(".meta"): + # meta_file = f + # continue + if f.endswith(".zseq"): + seq_file = f + continue + if f.endswith(".zbank"): + zbank_file = f + continue + if f.endswith(".bankmeta"): + bankmeta_file = f + continue + + if not seq_file: + raise FileNotFoundError(f'No .seq file in: "{file_name}". This should never happen') + if zbank_file and not bankmeta_file: + raise FileNotFoundError(f'Custom track "{file_name}" contains .zbank but no .bankmeta') + type_match = False + if 'categories.txt' in zip.namelist(): + mmrs_categories_txt = zip.read('categories.txt').decode() + delimitingChar: str = ',' + if '-' in mmrs_categories_txt: # MMR is ridiculous... + delimitingChar = '-' + elif '\n' in mmrs_categories_txt: + delimitingChar = '\n' + mmrs_categories = mmrs_categories_txt.split(delimitingChar) + if seq_type.lower() == 'fanfare' and ('8' in mmrs_categories or '9' in mmrs_categories or '10' in mmrs_categories): + type_match = True + elif seq_type.lower() == 'bgm' and not ('8' in mmrs_categories or '9' in mmrs_categories or '10' in mmrs_categories): + type_match = True + else: + raise Exception("OWL LIED TO ME") + + if type_match: + if zbank_file: + instrument_set = '-' + else: + instrument_set = int(seq_file.split(".zseq")[0], 16) # Get the instrument set from the .zseq file name + cosmetic_name = filepath + # Create new sequence + seq = Sequence(filepath, cosmetic_name, seq_type=seq_type, seq_file = seq_file, instrument_set = instrument_set) + if zbank_file: + seq.zbank_file = zbank_file + seq.bankmeta = bankmeta_file + seq.zsounds = [] + + seq.game = SequenceGame.MM + seq.new_instrument_set = True # MM sequences always require new instrument set. + # Make sure we have the MM audio binaries + if not os.path.exists(os.path.join(data_path(), 'Music', 'MM.audiobin')): + # Raise error. Maybe just skip and log a warning? + raise FileNotFoundError(".MMRS sequence found but missing MM.audiobin") + + for f in zip.namelist(): + if f.lower().endswith(".zsound"): + split: str = f.split('.zsound') + split = split[0].split('_') + zsound = { + 'type': None, + 'index': None, + 'alt': None, + 'file': f, + 'tempaddr': int(split[1], 16) + } + seq.zsounds.append(zsound) + return seq + return None + +def process_sequence_ootrs(filepath: str, file_name: str, seq_type: str, include_custom_audiobanks: bool, groups) -> Sequence: + with zipfile.ZipFile(filepath) as zip: + # Make sure meta file and seq file exists + meta_file = None + seq_file = None + zbank_file = None + bankmeta_file = None + for f in zip.namelist(): + if f.endswith(".meta"): + meta_file = f + continue + if f.endswith(".seq") or f.endswith(".zseq"): + seq_file = f + continue + if f.endswith(".zbank"): + if not include_custom_audiobanks: # Check if we are excluding sequences with custom banks + return None + zbank_file = f + continue + if f.endswith(".bankmeta"): + bankmeta_file = f + continue + + if not meta_file: + raise FileNotFoundError(f'No .meta file in: "{file_name}". This should never happen') + if not seq_file: + raise FileNotFoundError(f'No .seq file in: "{file_name}". This should never happen') + if zbank_file and not bankmeta_file: + raise FileNotFoundError(f'Custom track "{file_name}" contains .zbank but no .bankmeta') + + instrument_set = '-' + cosmetic_name = filepath + type_match = False + + # Read meta info + with zip.open(meta_file, 'r') as stream: + lines = io.TextIOWrapper(stream).readlines() # Use TextIOWrapper in order to get text instead of binary from the seq. + # Strip newline(s) + lines = [line.rstrip() for line in lines] + cosmetic_name = lines[0] + instrument_set = lines[1] + if lines[2].lower() == seq_type.lower(): + type_match = True + if len(lines) >= 4: + seq_groups = lines[3].split(',') + for group in seq_groups: + group = group.strip() + if group not in groups: + groups[group] = [] + groups[group].append(cosmetic_name) + + if type_match: + # Create new sequence + seq = Sequence(filepath, cosmetic_name, seq_type=seq_type, seq_file = seq_file, instrument_set = instrument_set) + if zbank_file: + seq.zbank_file = zbank_file + seq.bankmeta = bankmeta_file + seq.zsounds = [] + + if seq.instrument_set < 0x00 or seq.instrument_set > 0x25: + raise Exception(f'{seq.name}: OOT Sequence instrument set must be in range [0x00, 0x25]') + if seq.cosmetic_name == "None": + raise Exception(f'{seq.name}: Sequences should not be named "None" as that is used for disabled music.') + + try: + # Process ZSOUND lines. Make these lines in the format of ZSOUND:file_path:temp_addr + for line in lines: + tokens = line.split(":") + if tokens[0] == "ZSOUND": + zsound = { + 'type': tokens[1], + 'index': int(tokens[2]), + 'alt': tokens[3], + 'file': tokens[4], + 'tempaddr': None + } + seq.zsounds.append(zsound) + except: + seq.zsounds.clear() + # Error occurred using the new way, try the old way + for line in lines: + tokens = line.split(":") + if tokens[0] == "ZSOUND": + zsound = { + 'type': None, + 'index': None, + 'alt': None, + 'file': tokens[1], + 'tempaddr': int(tokens[2], 16) + } + seq.zsounds.append(zsound) + return seq + return None diff --git a/Sequence.py b/Sequence.py new file mode 100644 index 000000000..95af48992 --- /dev/null +++ b/Sequence.py @@ -0,0 +1,42 @@ +from __future__ import annotations +from enum import Enum +from typing import TYPE_CHECKING, Optional, Any + + +class SequenceGame(Enum): + OOT = 1 + MM = 2 + +# Represents the information associated with a sequence, aside from the sequence data itself +class Sequence: + def __init__(self, name: str, cosmetic_name: str, seq_type: str, type: int = 0x0202, instrument_set: int | str = 0x03, + replaces: int = -1, vanilla_id: int = -1, seq_file: Optional[str] = None, new_instrument_set: bool = False, + zsounds: Optional[list[dict[str, str]]] = None) -> None: + self.name: str = name + self.seq_file = seq_file + self.cosmetic_name: str = cosmetic_name + self.replaces: int = replaces + self.vanilla_id: int = vanilla_id + self.seq_type = seq_type + self.type: int = type + self.new_instrument_set: bool = new_instrument_set + self.zsounds: Optional[list[dict[str, str]]] = zsounds + self.zbank_file: Optional[str] = None + self.bankmeta: Optional[str] = None + self.game: Optional[SequenceGame] = SequenceGame.OOT + + self.instrument_set: int = 0x0 + if isinstance(instrument_set, str): + if instrument_set == '-': + self.new_instrument_set = True + else: + self.instrument_set = int(instrument_set, 16) + else: + self.instrument_set = instrument_set + + def copy(self) -> Sequence: + copy = Sequence(self.name, self.cosmetic_name, self.seq_type, self.type, self.instrument_set, self.replaces, self.vanilla_id, self.seq_file, self.new_instrument_set, self.zsounds) + copy.zbank_file = self.zbank_file + copy.bankmeta = self.bankmeta + copy.game = self.game + return copy \ No newline at end of file diff --git a/Unittest.py b/Unittest.py index faec3a667..37bb1a2f3 100644 --- a/Unittest.py +++ b/Unittest.py @@ -3,6 +3,7 @@ # See `python -m unittest -h` or `pytest -h` for more options. from __future__ import annotations +import io import json import logging import os @@ -19,10 +20,18 @@ from ItemPool import remove_junk_items, remove_junk_ludicrous_items, ludicrous_items_base, ludicrous_items_extended, trade_items, ludicrous_exclusions from LocationList import location_is_viewable from Main import main, resolve_settings, build_world_graphs +<<<<<<< HEAD from Messages import Message, read_messages, shuffle_messages from Settings import Settings, get_preset_files from Spoiler import Spoiler from Rom import Rom +======= +from Messages import Message +from Rom import Rom +from Settings import Settings, get_preset_files +from Spoiler import Spoiler +from Audiobank import * +>>>>>>> 4a31aed4 (Add MMR custom music support and OOTRS improvements) test_dir = os.path.join(os.path.dirname(__file__), 'tests') output_dir = os.path.join(test_dir, 'Output') @@ -821,7 +830,7 @@ def test_presets(self): self.verify_woth(spoiler) self.verify_playthrough(spoiler) self.verify_disables(spoiler) - + # remove this to run the fuzzer @unittest.skip("generally slow and failures can be ignored") def test_fuzzer(self): @@ -855,6 +864,7 @@ def test_fuzzer(self): logging.getLogger('').exception(f'Failed to generate with these settings:\n{settings.get_settings_display()}\n') raise +<<<<<<< HEAD class TestTextShuffle(unittest.TestCase): def test_text_shuffle(self): if not os.path.isfile('./ZOOTDEC.z64'): @@ -863,3 +873,141 @@ def test_text_shuffle(self): messages = read_messages(rom) shuffle_messages(messages) shuffle_messages(messages, False) +======= +class TestCustomAudio(unittest.TestCase): + def test_audiobank(self): + AUDIOBANK_POINTER_TABLE = 0x00B896A0 + AUDIOBANK_ADDR = 0xD390 + AUDIOTABLE_INDEX_ADDR = 0xB8A1C0 + AUDIOTABLE_ADDR = 0x79470 + + + rom: Rom = Rom("ZOOTDEC.z64") + audiobank_file = rom.read_bytes(AUDIOBANK_ADDR, 0x1CA50) + audiotable_index = rom.read_bytes(AUDIOTABLE_INDEX_ADDR, 0x80) # Read audiotable index into bytearray + audiotable_file = rom.read_bytes(AUDIOTABLE_ADDR, 0x460AD0) # Read audiotable (samples) into bytearray + rom_bytes: bytearray = rom.buffer + audiobank_table_header = rom.read_bytes(AUDIOBANK_POINTER_TABLE, 0x10) + num_banks = int.from_bytes(audiobank_table_header[0:2]) + audiobanks: list[AudioBank] = [] + for i in range(0, num_banks): + curr_entry = rom.read_bytes(AUDIOBANK_POINTER_TABLE + 0x10 + (0x10 * i), 0x10) + audiobank: AudioBank = AudioBank(curr_entry, audiobank_file, audiotable_file, audiotable_index) + audiobanks.append(audiobank) + self.assertEqual(num_banks, 0x26) + self.assertEqual(audiobanks[0x25].bank_offset, 0x19110) + self.assertEqual(audiobanks[0x25].size, 0x3940) + + def test_mmrs_oot(self): + AUDIOBANK_POINTER_TABLE = 0x00B896A0 + AUDIOBANK_ADDR = 0xD390 + AUDIOTABLE_INDEX_ADDR = 0xB8A1C0 + AUDIOTABLE_ADDR = 0x79470 + + rom: Rom = Rom("ZOOTDEC.z64") + audiobank_file = rom.read_bytes(AUDIOBANK_ADDR, 0x1CA50) + audiotable_index = rom.read_bytes(AUDIOTABLE_INDEX_ADDR, 0x80) # Read audiotable index into bytearray + audiotable_file = rom.read_bytes(AUDIOTABLE_ADDR, 0x460AD0) # Read audiotable (samples) into bytearray + rom_bytes: bytearray = rom.buffer + audiobank_table_header = rom.read_bytes(AUDIOBANK_POINTER_TABLE, 0x10) + num_banks = int.from_bytes(audiobank_table_header[0:2]) + oot_audiobanks: list[AudioBank] = [] + for i in range(0, num_banks): + curr_entry = rom.read_bytes(AUDIOBANK_POINTER_TABLE + 0x10 + (0x10 * i), 0x10) + audiobank: AudioBank = AudioBank(curr_entry, audiobank_file, audiotable_file, audiotable_index) + oot_audiobanks.append(audiobank) + + mm_audiobank_index = open("data/Music/mm_audiobank_index.bin", 'rb').read() + mm_audiobank_file = open("data/Music/mm_audiobank.bin", 'rb').read() + mm_audiotable_index = open("data/Music/mm_audiotable_index.bin", 'rb').read() + mm_audiotable_file = open("data/Music/mm_audiotable.bin", 'rb').read() + num_banks = int.from_bytes(mm_audiobank_index[0:2]) + offset = 0x10 + mm_audiobanks: list[AudioBank] = [] + for i in range(0, num_banks): + bank_entry = mm_audiobank_index[offset:offset+0x10] + audiobank: AudioBank = AudioBank(bank_entry, mm_audiobank_file, mm_audiotable_file, mm_audiotable_index) + mm_audiobanks.append(audiobank) + offset += 0x10 + + from zipfile import ZipFile, ZipInfo + + mmrs = ZipFile("data/Music/inverted stone tower.zip", 'r') + newbank_data = None + bankmeta = None + meta = None + for file in mmrs.filelist: + if file.filename.endswith(".zbank"): + newbank_data = bytearray(mmrs.read(file)) + if file.filename.endswith(".bankmeta"): + bankmeta = mmrs.read(file) + if file.filename.endswith(".meta"): + with mmrs.open(file, 'r') as f: + meta = io.TextIOWrapper(f).readlines() + + zsounds = [] + # Process the zsounds out of the meta file + for line in meta: + if line.startswith("ZSOUND"): + line = line.rstrip() + zsound = {} + split = line.split(':') + zsound['type'] = split[1] + zsound['index'] = int(split[2]) + zsound['alt'] = split[3] + zsound['filename'] = split[4] + zsounds.append(zsound) + + if newbank_data and bankmeta: + bank_entry = bytearray(4) + len(newbank_data).to_bytes(4, 'big') + bankmeta + newbank: AudioBank = AudioBank(bank_entry, newbank_data, mm_audiotable_file, mm_audiotable_index) + + mm_samples_to_add: list[Sample] = [] + # Cross reference MM instrument data against OOT and update accordingly + all_samples = newbank.get_all_samples() + all_samples = [sample for sample in all_samples if sample.addr != -1] + for sample in all_samples: + match = find_sample_in_audiobanks(oot_audiobanks, sample.data) + if match: # Found a matching sample in OOT. Just update the bank with the corresponding sample address + # Update the sample's offset in the new bank + newbank_data[sample.bank_offset+4:sample.bank_offset+8] = match.audiotable_addr.to_bytes(4, 'big') + else: # Didn't find a matching sample, need to add the data to the end of audiotable and update the bank. + mm_samples_to_add.append(sample) + + # Check for any drums that need to be updated (they will have -1 in their sample addresses) + for drum_id in range(0, len(newbank.drums)): + if newbank.drums[drum_id] and newbank.drums[drum_id].sample and newbank.drums[drum_id].sample.addr == -1: + for zsound in zsounds: + if zsound['type'] == 'DRUM' and zsound['index'] == drum_id: + newbank.drums[drum_id].sample.data = mmrs.read(zsound['filename']) + break + + # Check for any new SFX that need to be updated (they will have -1 in their sample addresses) + for sfx_id in range(0, len(newbank.SFX)): + if newbank.SFX[sfx_id] and newbank.SFX[sfx_id].sample and newbank.SFX[sfx_id].sample.addr == -1: + for zsound in zsounds: + if zsound['type'] == 'SFX' and zsound['index'] == sfx_id: + newbank.SFX[sfx_id].sample.data = mmrs.read(zsound['filename']) + break + + # Check if any instruments need to be updated (they will have -1 in their sample addresses) + for instr_id in range(0, len(newbank.instruments)): + if newbank.instruments[instr_id]: + instr = newbank.instruments[instr_id] + if instr.lowNoteSample and instr.lowNoteSample.addr == -1: + for zsound in zsounds: + if zsound['type'] == 'INST' and zsound['index'] == instr_id and zsound['alt'] == 'LOW': + instr.lowNoteSample.data = mmrs.read(zsound['filename']) + break + if instr.normalNoteSample and instr.normalNoteSample.addr == -1: + for zsound in zsounds: + if zsound['type'] == 'INST' and zsound['index'] == instr_id and zsound['alt'] == 'NORM': + instr.normalNoteSample.data = mmrs.read(zsound['filename']) + break + if instr.highNoteSample and instr.highNoteSample.addr == -1: + for zsound in zsounds: + if zsound['type'] == 'INST' and zsound['index'] == instr_id and zsound['alt'] == 'HIGH': + instr.highNoteSample.data = mmrs.read(zsound['filename']) + break + +>>>>>>> 4a31aed4 (Add MMR custom music support and OOTRS improvements) diff --git a/data/Music/update_ootrs_v2.py b/data/Music/update_ootrs_v2.py new file mode 100644 index 000000000..b42652257 --- /dev/null +++ b/data/Music/update_ootrs_v2.py @@ -0,0 +1,308 @@ +import io +import shutil +from zipfile import * +import os +from io import FileIO + +# Container for storing Audiotable, Audiobank, Audiotable_index, Audiobank_index +class Audiobin: + def __init__(self, _Audiobank: bytearray, _Audiobank_index: bytearray, _Audiotable: bytearray, _Audiotable_index: bytearray): + self.Audiobank: bytearray = _Audiobank + self.Audiobank_index: bytearray = _Audiobank_index + self.Audiotable: bytearray = _Audiotable + self.Audiotable_index: bytearray = _Audiotable_index + + num_banks = int.from_bytes(self.Audiobank_index[0:2]) + self.audiobanks: list[AudioBank] = [] + for i in range(0, num_banks): + index = 0x10 + (0x10*i) + curr_entry = self.Audiobank_index[index:index+0x10] + audiobank: AudioBank = AudioBank(curr_entry, self.Audiobank, self.Audiotable, self.Audiotable_index) + self.audiobanks.append(audiobank) + + def find_sample_in_audiobanks(self, sample_data: bytearray): + for audiobank in self.audiobanks: + for drum in audiobank.drums: + if drum and drum.sample: + if drum.sample.data == sample_data: + return drum.sample + for instrument in audiobank.instruments: + if instrument: + if instrument.highNoteSample and instrument.highNoteSample.data == sample_data: + return instrument.highNoteSample + if instrument.lowNoteSample and instrument.lowNoteSample.data == sample_data: + return instrument.lowNoteSample + if instrument.normalNoteSample and instrument.normalNoteSample.data == sample_data: + return instrument.normalNoteSample + for sfx in audiobank.SFX: + if sfx and sfx.sample: + if sfx.sample.data == sample_data: + return sfx.sample + return None + +class Sample: + def __init__(self, bankdata: bytearray, audiotable_file: bytearray, audiotable_index: bytearray, sample_offset: int, audiotable_id: int, parent): + # Process the sample + self.parent = parent + self.bank_offset = sample_offset + self.sample_header = bankdata[sample_offset:sample_offset + 0x10] + self.codec = (self.sample_header[0] & 0xF0) >> 4 + self.medium = (self.sample_header[0] & 0x0C) >> 2 + self.size = int.from_bytes(self.sample_header[1:4]) + self.addr = int.from_bytes(self.sample_header[4:8]) + + if audiotable_file and self.addr > len(audiotable_file): # The offset is higher than the total size of audiotable so we'll assume it doesn't actually exist. # We'll need to get the sample data from ZSOUND files in the archive. + self.data = None + self.addr = -1 + return + # Read the audiotable pointer table entry + if audiotable_file and audiotable_index: + audiotable_index_offset = 0x10 + (audiotable_id * 0x10) + audiotable_entry = audiotable_index[audiotable_index_offset:audiotable_index_offset + 0x10] + audiotable_offset = int.from_bytes(audiotable_entry[0:4]) + sample_address = audiotable_offset + self.addr + self.audiotable_addr = sample_address + # Read the sample data + self.data = audiotable_file[sample_address:sample_address+self.size] + else: + self.audiotable_addr = -1 + self.data = None + +# Loads an audiobank and it's corresponding instrument/drum/sfxs +class AudioBank: + + # Constructor: + # table_entry - 0x10 byte audiobank entry which contains info like the bank offset, size, number of instruments, etc. + # audiobank_file - the Audiobank file as a byte array + # audiotable_file - the Audiotable file as a byte array + # audiotable_index - the Audiotable index (pointer table) which provides an offsets into the Audiotable file where a bank's instrument samples offsets are calculated from. + def __init__(self, table_entry: bytearray, audiobank_file: bytearray, audiotable_file: bytearray, audiotable_index: bytearray) -> None: + + # Process bank entry + self.bank_offset: int = int.from_bytes(table_entry[0:4]) # Offset of the bank in the Audiobank file + self.size: int = int.from_bytes(table_entry[4:8]) # Size of the bank, in bytes + self.load_location: int = table_entry[8] # ROM/RAM/DISK + self.type: int = table_entry[9] + self.audiotable_id: int = table_entry[10] # Read audiotable id from the table entry. Instrument data offsets are in relation to this + self.unk: int = table_entry[11] # 0xFF + self.num_instruments: int = table_entry[12] + self.num_drums: int = table_entry[13] + self.num_sfx: int = int.from_bytes(table_entry[14:16]) + self.bank_data = audiobank_file[self.bank_offset:self.bank_offset + self.size] + self.original_data = self.bank_data.copy() + self.table_entry = table_entry + # Process the bank + + # Read drums + self.drums: list[Drum] = [] + drum_offset = int.from_bytes(self.bank_data[0:4]) # Get the drum pointer. This is the first uint32 in the bank. Points to a list of drum offsets of length num_drums + for i in range(0, self.num_drums): # Read each drum + offset = drum_offset + 4*i + offset = int.from_bytes(self.bank_data[offset:offset+4]) + drum = Drum(i, self.bank_data, audiotable_file, audiotable_index, offset, self.audiotable_id) if offset != 0 else None + self.drums.append(drum) + + # Read SFX + self.SFX: list[SFX] = [] + sfx_offset = int.from_bytes(self.bank_data[4:8]) # Get the SFX pointer. this is the second uint32 in the bank. Points to a list of Sound objects which are 8 bytes each (Sample offsets + tuning) + for i in range(0, self.num_sfx): # Read each SFX + offset = sfx_offset + 8*i + sfx = SFX(i, self.bank_data, audiotable_file, audiotable_index, offset, self.audiotable_id) if offset != 0 else None + self.SFX.append(sfx) + + self.instruments: list[Instrument] = [] + # Read the instruments + for i in range(0, self.num_instruments): + offset = 0x08 + 4*i + instr_offset = int.from_bytes(self.bank_data[offset:offset+4]) + instrument: Instrument = Instrument(i, self.bank_data, audiotable_file, audiotable_index, instr_offset, self.audiotable_id) if instr_offset != 0 else None + self.instruments.append(instrument) + + + def __str__(self): + return "Offset: " + hex(self.bank_offset) + ", " + "Len:" + hex(self.size) + + def get_all_samples(self) -> list[Sample]: + all_sounds = self.drums + self.instruments + self.SFX + all_samples: list[Sample] = [] + for sound in all_sounds: + if type(sound) == Instrument: + instrument: Instrument = sound + if instrument.highNoteSample: + all_samples.append(instrument.highNoteSample) + if instrument.lowNoteSample: + all_samples.append(instrument.lowNoteSample) + if instrument.normalNoteSample: + all_samples.append(instrument.normalNoteSample) + + elif type(sound) == Drum: + drum: Drum = sound + if drum.sample: + all_samples.append(drum.sample) + elif type(sound) == SFX: + sfx: SFX = sound + if sfx.sample: + all_samples.append(sfx.sample) + return all_samples + + def build_entry(self, offset: int) -> bytes: + bank_entry: bytes = offset.to_bytes(4, 'big') + bank_entry += len(self.bank_data).to_bytes(4, 'big') + bank_entry += self.table_entry[8:16] + return bank_entry + +class Drum: + def __init__(self, drum_id: int, bankdata: bytearray, audiotable_file: bytearray, audiotable_index: bytearray, drum_offset: int, audiotable_id: int) -> None: + self.drum_id = drum_id + self.releaseRate = bankdata[drum_offset] + self.pan = bankdata[drum_offset + 1] + self.sampleOffset = int.from_bytes(bankdata[drum_offset + 4:drum_offset+8]) + self.sampleTuning = int.from_bytes(bankdata[drum_offset + 8:drum_offset+12]) + self.envelopePointOffset = int.from_bytes(bankdata[drum_offset+12:drum_offset+16]) + self.sample: Sample = Sample(bankdata, audiotable_file, audiotable_index, self.sampleOffset, audiotable_id, self) + +class SFX: + def __init__(self, sfx_id: int, bankdata: bytearray, audiotable_file: bytearray, audiotable_index: bytearray, sfx_offset: int, audiotable_id: int) -> None: + self.sfx_id = sfx_id + self.sampleOffset = int.from_bytes(bankdata[sfx_offset:sfx_offset+4]) + self.sampleTuning = int.from_bytes(bankdata[sfx_offset+4:sfx_offset+8]) + self.sample: Sample = Sample(bankdata, audiotable_file, audiotable_index, self.sampleOffset, audiotable_id, self) + + +class Instrument: + def __init__(self, inst_id: int, bankdata: bytearray, audiotable_file: bytearray, audiotable_index: bytearray, instr_offset: int, audiotable_id: int) -> None: + self.inst_id = inst_id + self.normalRangeLo = bankdata[instr_offset + 1] + self.normalRangeHi = bankdata[instr_offset + 2] + self.releaseRate = bankdata[instr_offset + 3] + self.AdsrEnvelopePointOffset = int.from_bytes(bankdata[instr_offset + 4:instr_offset+8]) + self.lowNoteSampleOffset = int.from_bytes(bankdata[instr_offset + 8:instr_offset+12]) + self.lowNoteTuning = int.from_bytes(bankdata[instr_offset + 12: instr_offset + 16]) + self.normalNoteSampleOffset = int.from_bytes(bankdata[instr_offset + 16:instr_offset+20]) + self.normalNoteTuning = int.from_bytes(bankdata[instr_offset + 20:instr_offset + 24]) + self.highNoteSampleOffset = int.from_bytes(bankdata[instr_offset + 24:instr_offset+28]) + self.highNoteSampleTuning = int.from_bytes(bankdata[instr_offset + 28:instr_offset+32]) + self.lowNoteSample: Sample = Sample(bankdata, audiotable_file, audiotable_index, self.lowNoteSampleOffset, audiotable_id, self) if self.lowNoteSampleOffset != 0 else None + self.normalNoteSample: Sample = Sample(bankdata, audiotable_file, audiotable_index, self.normalNoteSampleOffset, audiotable_id, self) if self.normalNoteSampleOffset != 0 else None + self.highNoteSample: Sample = Sample(bankdata, audiotable_file, audiotable_index, self.highNoteSampleOffset, audiotable_id, self) if self.highNoteSampleOffset != 0 else None + +def update_ootrs_v2(filename: str, outfilename: str): + with ZipFile(filename, 'r') as zip: + # Look for zbank and meta + zbank = None + meta_lines: list[str] = [] + bankmeta = None + zbank_name = None + meta_name = None + for file in zip.namelist(): + if file.endswith(".zbank"): + zbank_name = file + zbank = bytearray(zip.read(file)) + if file.endswith(".meta"): + meta_name = file + with zip.open(file, 'r') as stream: + meta_lines = io.TextIOWrapper(stream).readlines() # Use TextIOWrapper in order to get text instead of binary from the seq. + # Strip newline(s) + meta_lines = [line.rstrip() for line in meta_lines] + if file.endswith(".bankmeta"): + bankmeta = zip.read(file) + if zbank: + + zsounds = [] + for line in meta_lines: + if line.startswith("ZSOUND"): + tokens = line.split(':') + zsound = { + 'file':tokens[1], + 'addr':int(tokens[2], 16) + } + zsounds.append(zsound) + tableentry = bytearray(4) + len(zbank).to_bytes(4, 'big') + bankmeta + bank = AudioBank(tableentry, zbank, None, None) + for zsound in zsounds: + for sample in bank.get_all_samples(): + if sample.addr == zsound['addr']: # Found the right sample + # Update the zbank addr to 0xFFFFFFFF + offset = sample.bank_offset + bank.bank_data[offset+4:offset+8] = (0xFFFFFFFF).to_bytes(4, 'big') + if type(sample.parent) == Instrument: + zsound['type'] = "INST" + zsound['id'] = sample.parent.inst_id + if sample == sample.parent.highNoteSample: + zsound['alt'] = 'HIGH' + elif sample == sample.parent.normalNoteSample: + zsound['alt'] = 'NORM' + elif sample == sample.parent.lowNoteSample: + zsound['alt'] = 'LOW' + elif type(sample.parent) == Drum: + zsound['type'] = "DRUM" + zsound['id'] = sample.parent.drum_id + zsound['alt'] = '' + elif type(sample.parent) == SFX: + zsound['type'] = "SFX" + zsound['id'] = sample.parent.sfx_id + zsound['alt'] = '' + break + if len(zsounds) > 0: + # Create new file + newzip = ZipFile(outfilename, 'w', compression=ZIP_DEFLATED, compresslevel=1) + for file in zip.namelist(): + # Don't copy the .zbank or .meta + if file.endswith(".zbank") or file.endswith(".meta"): + continue + buffer = zip.read(file) + newzip.writestr(file, buffer) + # Write the new zbank + newzip.writestr(zbank_name, bank.bank_data) + # Build new meta file + new_meta = "" + zsound_index = 0 + for line in meta_lines: + if not line.startswith("ZSOUND"): + new_meta += line + "\r\n" + else: + zsound_line = "ZSOUND:" + zsounds[zsound_index]['type'] + ":" + str(zsounds[zsound_index]['id']) + ":" + zsounds[zsound_index]['alt'] + ":" + zsounds[zsound_index]['file'] + new_meta += zsound_line + "\r\n" + zsound_index += 1 + newzip.writestr(meta_name, new_meta) + newzip.close() + return 1 + # If we got here, then we didn't update it, so just copy instead + shutil.copy2(filename, outfilename) + return 0 + +def update_all(): + directory = os.getcwd() + fixed_directory = os.path.join(directory, "fixed") + if not os.path.exists(fixed_directory): + os.mkdir(fixed_directory) + num_fixed = 0 + num_copied = 0 + for dirpath, _, filenames in os.walk(directory, followlinks=True): + relative_path = os.path.relpath(dirpath, directory) + new_dir_path = os.path.join(fixed_directory, relative_path) + # Create the corresponding subdirectory in the destination directory if it doesn't exist + if not os.path.exists(new_dir_path): + os.makedirs(new_dir_path) + for fname in filenames: + if fname.endswith(".ootrs"): + print("Fixing " + fname + "... ", end='') + filepath = os.path.join(dirpath, fname) + newpath = os.path.join(new_dir_path, fname) + try: + res = update_ootrs_v2(filepath, newpath) + if res == 1: + print("FIXED!") + num_fixed += 1 + elif res == 0: + print("COPIED") + num_copied += 1 + else: + print("UH OH") + except Exception as e: + if os.path.exists(newpath): + os.remove(newpath) + print(e) + print("Complete! Fixed: " + str(num_fixed) + ", Copied: " + str(num_copied)) + +update_all() \ No newline at end of file From a53122cff833995eacc4e474e210bfcf1b97800b Mon Sep 17 00:00:00 2001 From: Rob Realmuto Date: Tue, 10 Oct 2023 16:42:31 -0400 Subject: [PATCH 02/10] Fix drums/sfx in custom music not importing correctly --- Music.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Music.py b/Music.py index 5d8407b93..514b13f85 100644 --- a/Music.py +++ b/Music.py @@ -523,9 +523,9 @@ def rebuild_sequences(rom: Rom, sequences: list[Sequence], log: CosmeticsLog, sy elif parent.lowNoteSample and parent.lowNoteSample.addr == sample.addr: sample = newbank.instruments[parent.inst_id].lowNoteSample if type(parent) == Drum: - sample = newbank.SFX[parent.drum_id] + sample = newbank.drums[parent.drum_id].sample if type(parent) == SFX: - sample = newbank.SFX[parent.sfx_id] + sample = newbank.SFX[parent.sfx_id].sample sample.data = zip.read(zsound['file']) sample.addr = -1 # Set the sample address to -1 so that we know it's from a zsound zsound_samples.append(sample) From 7e63b7d6646811b83b3c7bd139064b73d02471ab Mon Sep 17 00:00:00 2001 From: Rob Realmuto Date: Fri, 20 Oct 2023 20:54:04 -0400 Subject: [PATCH 03/10] Fix .ootrs without bgm/fanfare designation --- MusicHelpers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/MusicHelpers.py b/MusicHelpers.py index ac092badf..3896bc597 100644 --- a/MusicHelpers.py +++ b/MusicHelpers.py @@ -155,7 +155,9 @@ def process_sequence_ootrs(filepath: str, file_name: str, seq_type: str, include lines = [line.rstrip() for line in lines] cosmetic_name = lines[0] instrument_set = lines[1] - if lines[2].lower() == seq_type.lower(): + if len(lines) < 3 and seq_type == 'bgm': + type_match = True + elif lines[2].lower() == seq_type.lower(): type_match = True if len(lines) >= 4: seq_groups = lines[3].split(',') From fa8956e599c34d5e110e4da4c6b0a51af5962d3d Mon Sep 17 00:00:00 2001 From: Rob Realmuto Date: Fri, 20 Oct 2023 21:00:07 -0400 Subject: [PATCH 04/10] Fix again --- MusicHelpers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/MusicHelpers.py b/MusicHelpers.py index 3896bc597..7c54dcdbe 100644 --- a/MusicHelpers.py +++ b/MusicHelpers.py @@ -157,8 +157,9 @@ def process_sequence_ootrs(filepath: str, file_name: str, seq_type: str, include instrument_set = lines[1] if len(lines) < 3 and seq_type == 'bgm': type_match = True - elif lines[2].lower() == seq_type.lower(): + elif len(lines) >= 3 and lines[2].lower() == seq_type.lower(): type_match = True + if len(lines) >= 4: seq_groups = lines[3].split(',') for group in seq_groups: From c1d23dc0d47ce592f22da41ca635bd935dfbf35e Mon Sep 17 00:00:00 2001 From: Rob Realmuto Date: Sat, 21 Oct 2023 19:28:01 -0400 Subject: [PATCH 05/10] Log sequence processing errors to cosmetics log instead of crashing --- Music.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/Music.py b/Music.py index 514b13f85..9bf284e9b 100644 --- a/Music.py +++ b/Music.py @@ -132,7 +132,8 @@ def __init__(self) -> None: def process_sequences(rom: Rom, ids: Iterable[tuple[str, int]], seq_type: str = 'bgm', disabled_source_sequences: Optional[list[str]] = None, disabled_target_sequences: Optional[dict[str, tuple[str, int]]] = None, include_custom: bool = True, sequences: Optional[dict[str, Sequence]] = None, target_sequences: Optional[dict[str, Sequence]] = None, - groups: Optional[dict[str, list[str]]] = None, include_custom_audiobanks: bool = False) -> tuple[dict[str, Sequence], dict[str, Sequence], dict[str, list[str]]]: + groups: Optional[dict[str, list[str]]] = None, include_custom_audiobanks: bool = False, + log: CosmeticsLog = None) -> tuple[dict[str, Sequence], dict[str, Sequence], dict[str, list[str]]]: disabled_source_sequences = [] if disabled_source_sequences is None else disabled_source_sequences disabled_target_sequences = {} if disabled_target_sequences is None else disabled_target_sequences sequences = {} if sequences is None else sequences @@ -185,14 +186,18 @@ def process_sequences(rom: Rom, ids: Iterable[tuple[str, int]], seq_type: str = filepath = os.path.join(dirpath, fname) seq = None - if fname.lower().endswith('.ootrs'): - seq = process_sequence_ootrs(filepath, fname, seq_type, include_custom_audiobanks, groups) - elif fname.lower().endswith('.zseq'): - seq = process_sequence_mmr_zseq(filepath, fname, seq_type, include_custom_audiobanks, groups) - elif fname.lower().endswith('.mmrs'): - seq = process_sequence_mmrs(filepath, fname, seq_type, include_custom_audiobanks, groups) - if seq: - sequences[seq.cosmetic_name] = seq + try: + if fname.lower().endswith('.ootrs'): + seq = process_sequence_ootrs(filepath, fname, seq_type, include_custom_audiobanks, groups) + elif fname.lower().endswith('.zseq'): + seq = process_sequence_mmr_zseq(filepath, fname, seq_type, include_custom_audiobanks, groups) + elif fname.lower().endswith('.mmrs'): + seq = process_sequence_mmrs(filepath, fname, seq_type, include_custom_audiobanks, groups) + if seq: + sequences[seq.cosmetic_name] = seq + except Exception as e: + if log: + log.errors.append(f"Error processing custom sequence {fname} - {e}") continue return sequences, target_sequences, groups @@ -754,7 +759,7 @@ def randomize_music(rom: Rom, settings: Settings, log: CosmeticsLog, symbols: di # Grab our lists of sequences. if settings.background_music in ['random', 'random_custom_only'] or bgm_mapped: - sequences, target_sequences, bgm_groups = process_sequences(rom, bgm_ids.values(), 'bgm', disabled_source_sequences, disabled_target_sequences, custom_sequences_enabled, include_custom_audiobanks=custom_audiobanks_enabled) + sequences, target_sequences, bgm_groups = process_sequences(rom, bgm_ids.values(), 'bgm', disabled_source_sequences, disabled_target_sequences, custom_sequences_enabled, include_custom_audiobanks=custom_audiobanks_enabled, log=log) if settings.background_music == 'random_custom_only': sequences = {name: seq for name, seq in sequences.items() if name not in bgm_ids or name in music_mapping.values()} if available_sequences: @@ -762,7 +767,7 @@ def randomize_music(rom: Rom, settings: Settings, log: CosmeticsLog, symbols: di sequences[sequence_name] = Sequence(sequence_name, sequence_name) if settings.fanfares in ['random', 'random_custom_only'] or ff_mapped or ocarina_mapped: - fanfare_sequences, target_fanfare_sequences, fanfare_groups = process_sequences(rom, ff_ids.values(), 'fanfare', disabled_source_sequences, disabled_target_sequences, custom_sequences_enabled, include_custom_audiobanks=custom_audiobanks_enabled) + fanfare_sequences, target_fanfare_sequences, fanfare_groups = process_sequences(rom, ff_ids.values(), 'fanfare', disabled_source_sequences, disabled_target_sequences, custom_sequences_enabled, include_custom_audiobanks=custom_audiobanks_enabled, log=log) if settings.fanfares == 'random_custom_only': fanfare_sequences = {name: seq for name, seq in fanfare_sequences.items() if name not in ff_ids or name in music_mapping.values()} if available_sequences: From c3d55bbdbf137296e19edf88f6c680240f4cb5d5 Mon Sep 17 00:00:00 2001 From: Rob Realmuto Date: Wed, 29 Nov 2023 09:32:02 -0500 Subject: [PATCH 06/10] Fix unit tests --- Unittest.py | 128 ++-------------------------------------------------- 1 file changed, 4 insertions(+), 124 deletions(-) diff --git a/Unittest.py b/Unittest.py index 37bb1a2f3..74e18be1f 100644 --- a/Unittest.py +++ b/Unittest.py @@ -20,18 +20,11 @@ from ItemPool import remove_junk_items, remove_junk_ludicrous_items, ludicrous_items_base, ludicrous_items_extended, trade_items, ludicrous_exclusions from LocationList import location_is_viewable from Main import main, resolve_settings, build_world_graphs -<<<<<<< HEAD from Messages import Message, read_messages, shuffle_messages from Settings import Settings, get_preset_files from Spoiler import Spoiler from Rom import Rom -======= -from Messages import Message -from Rom import Rom -from Settings import Settings, get_preset_files -from Spoiler import Spoiler from Audiobank import * ->>>>>>> 4a31aed4 (Add MMR custom music support and OOTRS improvements) test_dir = os.path.join(os.path.dirname(__file__), 'tests') output_dir = os.path.join(test_dir, 'Output') @@ -864,7 +857,6 @@ def test_fuzzer(self): logging.getLogger('').exception(f'Failed to generate with these settings:\n{settings.get_settings_display()}\n') raise -<<<<<<< HEAD class TestTextShuffle(unittest.TestCase): def test_text_shuffle(self): if not os.path.isfile('./ZOOTDEC.z64'): @@ -873,7 +865,7 @@ def test_text_shuffle(self): messages = read_messages(rom) shuffle_messages(messages) shuffle_messages(messages, False) -======= + class TestCustomAudio(unittest.TestCase): def test_audiobank(self): AUDIOBANK_POINTER_TABLE = 0x00B896A0 @@ -881,6 +873,8 @@ def test_audiobank(self): AUDIOTABLE_INDEX_ADDR = 0xB8A1C0 AUDIOTABLE_ADDR = 0x79470 + if not os.path.isfile('./ZOOTDEC.z64'): + self.skipTest("Base ROM file not available.") rom: Rom = Rom("ZOOTDEC.z64") audiobank_file = rom.read_bytes(AUDIOBANK_ADDR, 0x1CA50) @@ -888,7 +882,7 @@ def test_audiobank(self): audiotable_file = rom.read_bytes(AUDIOTABLE_ADDR, 0x460AD0) # Read audiotable (samples) into bytearray rom_bytes: bytearray = rom.buffer audiobank_table_header = rom.read_bytes(AUDIOBANK_POINTER_TABLE, 0x10) - num_banks = int.from_bytes(audiobank_table_header[0:2]) + num_banks = int.from_bytes(audiobank_table_header[0:2], 'big') audiobanks: list[AudioBank] = [] for i in range(0, num_banks): curr_entry = rom.read_bytes(AUDIOBANK_POINTER_TABLE + 0x10 + (0x10 * i), 0x10) @@ -897,117 +891,3 @@ def test_audiobank(self): self.assertEqual(num_banks, 0x26) self.assertEqual(audiobanks[0x25].bank_offset, 0x19110) self.assertEqual(audiobanks[0x25].size, 0x3940) - - def test_mmrs_oot(self): - AUDIOBANK_POINTER_TABLE = 0x00B896A0 - AUDIOBANK_ADDR = 0xD390 - AUDIOTABLE_INDEX_ADDR = 0xB8A1C0 - AUDIOTABLE_ADDR = 0x79470 - - rom: Rom = Rom("ZOOTDEC.z64") - audiobank_file = rom.read_bytes(AUDIOBANK_ADDR, 0x1CA50) - audiotable_index = rom.read_bytes(AUDIOTABLE_INDEX_ADDR, 0x80) # Read audiotable index into bytearray - audiotable_file = rom.read_bytes(AUDIOTABLE_ADDR, 0x460AD0) # Read audiotable (samples) into bytearray - rom_bytes: bytearray = rom.buffer - audiobank_table_header = rom.read_bytes(AUDIOBANK_POINTER_TABLE, 0x10) - num_banks = int.from_bytes(audiobank_table_header[0:2]) - oot_audiobanks: list[AudioBank] = [] - for i in range(0, num_banks): - curr_entry = rom.read_bytes(AUDIOBANK_POINTER_TABLE + 0x10 + (0x10 * i), 0x10) - audiobank: AudioBank = AudioBank(curr_entry, audiobank_file, audiotable_file, audiotable_index) - oot_audiobanks.append(audiobank) - - mm_audiobank_index = open("data/Music/mm_audiobank_index.bin", 'rb').read() - mm_audiobank_file = open("data/Music/mm_audiobank.bin", 'rb').read() - mm_audiotable_index = open("data/Music/mm_audiotable_index.bin", 'rb').read() - mm_audiotable_file = open("data/Music/mm_audiotable.bin", 'rb').read() - num_banks = int.from_bytes(mm_audiobank_index[0:2]) - offset = 0x10 - mm_audiobanks: list[AudioBank] = [] - for i in range(0, num_banks): - bank_entry = mm_audiobank_index[offset:offset+0x10] - audiobank: AudioBank = AudioBank(bank_entry, mm_audiobank_file, mm_audiotable_file, mm_audiotable_index) - mm_audiobanks.append(audiobank) - offset += 0x10 - - from zipfile import ZipFile, ZipInfo - - mmrs = ZipFile("data/Music/inverted stone tower.zip", 'r') - newbank_data = None - bankmeta = None - meta = None - for file in mmrs.filelist: - if file.filename.endswith(".zbank"): - newbank_data = bytearray(mmrs.read(file)) - if file.filename.endswith(".bankmeta"): - bankmeta = mmrs.read(file) - if file.filename.endswith(".meta"): - with mmrs.open(file, 'r') as f: - meta = io.TextIOWrapper(f).readlines() - - zsounds = [] - # Process the zsounds out of the meta file - for line in meta: - if line.startswith("ZSOUND"): - line = line.rstrip() - zsound = {} - split = line.split(':') - zsound['type'] = split[1] - zsound['index'] = int(split[2]) - zsound['alt'] = split[3] - zsound['filename'] = split[4] - zsounds.append(zsound) - - if newbank_data and bankmeta: - bank_entry = bytearray(4) + len(newbank_data).to_bytes(4, 'big') + bankmeta - newbank: AudioBank = AudioBank(bank_entry, newbank_data, mm_audiotable_file, mm_audiotable_index) - - mm_samples_to_add: list[Sample] = [] - # Cross reference MM instrument data against OOT and update accordingly - all_samples = newbank.get_all_samples() - all_samples = [sample for sample in all_samples if sample.addr != -1] - for sample in all_samples: - match = find_sample_in_audiobanks(oot_audiobanks, sample.data) - if match: # Found a matching sample in OOT. Just update the bank with the corresponding sample address - # Update the sample's offset in the new bank - newbank_data[sample.bank_offset+4:sample.bank_offset+8] = match.audiotable_addr.to_bytes(4, 'big') - else: # Didn't find a matching sample, need to add the data to the end of audiotable and update the bank. - mm_samples_to_add.append(sample) - - # Check for any drums that need to be updated (they will have -1 in their sample addresses) - for drum_id in range(0, len(newbank.drums)): - if newbank.drums[drum_id] and newbank.drums[drum_id].sample and newbank.drums[drum_id].sample.addr == -1: - for zsound in zsounds: - if zsound['type'] == 'DRUM' and zsound['index'] == drum_id: - newbank.drums[drum_id].sample.data = mmrs.read(zsound['filename']) - break - - # Check for any new SFX that need to be updated (they will have -1 in their sample addresses) - for sfx_id in range(0, len(newbank.SFX)): - if newbank.SFX[sfx_id] and newbank.SFX[sfx_id].sample and newbank.SFX[sfx_id].sample.addr == -1: - for zsound in zsounds: - if zsound['type'] == 'SFX' and zsound['index'] == sfx_id: - newbank.SFX[sfx_id].sample.data = mmrs.read(zsound['filename']) - break - - # Check if any instruments need to be updated (they will have -1 in their sample addresses) - for instr_id in range(0, len(newbank.instruments)): - if newbank.instruments[instr_id]: - instr = newbank.instruments[instr_id] - if instr.lowNoteSample and instr.lowNoteSample.addr == -1: - for zsound in zsounds: - if zsound['type'] == 'INST' and zsound['index'] == instr_id and zsound['alt'] == 'LOW': - instr.lowNoteSample.data = mmrs.read(zsound['filename']) - break - if instr.normalNoteSample and instr.normalNoteSample.addr == -1: - for zsound in zsounds: - if zsound['type'] == 'INST' and zsound['index'] == instr_id and zsound['alt'] == 'NORM': - instr.normalNoteSample.data = mmrs.read(zsound['filename']) - break - if instr.highNoteSample and instr.highNoteSample.addr == -1: - for zsound in zsounds: - if zsound['type'] == 'INST' and zsound['index'] == instr_id and zsound['alt'] == 'HIGH': - instr.highNoteSample.data = mmrs.read(zsound['filename']) - break - ->>>>>>> 4a31aed4 (Add MMR custom music support and OOTRS improvements) From fec101144ec86703eaa5a014a8d44b0e9713e1b4 Mon Sep 17 00:00:00 2001 From: Rob Realmuto Date: Sat, 10 Feb 2024 00:52:33 -0500 Subject: [PATCH 07/10] CI --- Audiobank.py | 7 +++---- Music.py | 8 ++++---- MusicHelpers.py | 16 ++++++++-------- Sequence.py | 2 +- Unittest.py | 4 ++-- 5 files changed, 18 insertions(+), 19 deletions(-) diff --git a/Audiobank.py b/Audiobank.py index 9a7bc5745..34790deee 100644 --- a/Audiobank.py +++ b/Audiobank.py @@ -118,7 +118,7 @@ def __init__(self, table_entry: bytearray, audiobank_file: bytearray, audiotable def __str__(self): return "Offset: " + hex(self.bank_offset) + ", " + "Len:" + hex(self.size) - + def get_all_samples(self) -> list[Sample]: all_sounds = self.drums + self.instruments + self.SFX all_samples: list[Sample] = [] @@ -131,7 +131,7 @@ def get_all_samples(self) -> list[Sample]: all_samples.append(instrument.lowNoteSample) if instrument.normalNoteSample: all_samples.append(instrument.normalNoteSample) - + elif type(sound) == Drum: drum: Drum = sound if drum.sample: @@ -164,7 +164,7 @@ def __init__(self, sfx_id: int, bankdata: bytearray, audiotable_file: bytearray, self.sampleOffset = int.from_bytes(bankdata[sfx_offset:sfx_offset+4], 'big') self.sampleTuning = int.from_bytes(bankdata[sfx_offset+4:sfx_offset+8], 'big') self.sample: Sample = Sample(bankdata, audiotable_file, audiotable_index, self.sampleOffset, audiotable_id, self) - + class Instrument: def __init__(self, inst_id: int, bankdata: bytearray, audiotable_file: bytearray, audiotable_index: bytearray, instr_offset: int, audiotable_id: int) -> None: @@ -182,4 +182,3 @@ def __init__(self, inst_id: int, bankdata: bytearray, audiotable_file: bytearray self.lowNoteSample: Sample = Sample(bankdata, audiotable_file, audiotable_index, self.lowNoteSampleOffset, audiotable_id, self) if self.lowNoteSampleOffset != 0 else None self.normalNoteSample: Sample = Sample(bankdata, audiotable_file, audiotable_index, self.normalNoteSampleOffset, audiotable_id, self) if self.normalNoteSampleOffset != 0 else None self.highNoteSample: Sample = Sample(bankdata, audiotable_file, audiotable_index, self.highNoteSampleOffset, audiotable_id, self) if self.highNoteSampleOffset != 0 else None - diff --git a/Music.py b/Music.py index 9bf284e9b..9467458b5 100644 --- a/Music.py +++ b/Music.py @@ -428,9 +428,9 @@ def rebuild_sequences(rom: Rom, sequences: list[Sequence], log: CosmeticsLog, sy audiobank = rom.read_bytes(AUDIOBANK_ADDR, 0x1CA50) audiotable_index = rom.read_bytes(AUDIOTABLE_INDEX_ADDR, 0x80) # Read audiotable index into bytearray audiotable = rom.read_bytes(AUDIOTABLE_ADDR, 0x460AD0) # Read audiotable (samples) into bytearray - + oot_audiobin: Audiobin = Audiobin(audiobank, audiobank_index, audiotable, audiotable_index) - + # Load MM Audiobin if needed mm_audiobin: Audiobin = None for i in range(0x6E): # Loop through all the replacement sequences @@ -455,7 +455,7 @@ def rebuild_sequences(rom: Rom, sequences: list[Sequence], log: CosmeticsLog, sy bankdata = None bank_entry = None audiobin = oot_audiobin if j.game == SequenceGame.OOT else mm_audiobin # Load the correct audiobin - + # Handle MMR .zseq files. These use vanilla MM banks so just build the bank using the data from the MM audiobin. We'll update it with OOT data later if j.name.lower().endswith('.zseq'): # Get the entry from Audiobank_index @@ -509,7 +509,7 @@ def rebuild_sequences(rom: Rom, sequences: list[Sequence], log: CosmeticsLog, sy else: newbank = AudioBank(bank_entry, bankdata, audiobin.Audiotable, audiobin.Audiotable_index) newbank.bank_index = new_bank_index - + zsound_samples: list[Sample] = [] # Handle new zsounds tempbank = AudioBank(bank_entry, bankdata, None, None) diff --git a/MusicHelpers.py b/MusicHelpers.py index 7c54dcdbe..8b85e1248 100644 --- a/MusicHelpers.py +++ b/MusicHelpers.py @@ -26,7 +26,7 @@ def process_sequence_mmr_zseq(filepath: str, file_name: str, seq_type: str, incl type_match = True elif seq_type.lower() == 'bgm' and not ('8' in mmrs_categories or '9' in mmrs_categories or '10' in mmrs_categories): type_match = True - + if not type_match: return None @@ -44,7 +44,7 @@ def process_sequence_mmrs(filepath: str, file_name: str, seq_type: str, include_ seq_file = None zbank_file = None bankmeta_file = None - for f in zip.namelist(): + for f in zip.namelist(): # Uncomment if MMR ever decides to wisen up and use a common format #if f.endswith(".meta"): # meta_file = f @@ -63,7 +63,7 @@ def process_sequence_mmrs(filepath: str, file_name: str, seq_type: str, include_ raise FileNotFoundError(f'No .seq file in: "{file_name}". This should never happen') if zbank_file and not bankmeta_file: raise FileNotFoundError(f'Custom track "{file_name}" contains .zbank but no .bankmeta') - type_match = False + type_match = False if 'categories.txt' in zip.namelist(): mmrs_categories_txt = zip.read('categories.txt').decode() delimitingChar: str = ',' @@ -78,7 +78,7 @@ def process_sequence_mmrs(filepath: str, file_name: str, seq_type: str, include_ type_match = True else: raise Exception("OWL LIED TO ME") - + if type_match: if zbank_file: instrument_set = '-' @@ -98,7 +98,7 @@ def process_sequence_mmrs(filepath: str, file_name: str, seq_type: str, include_ if not os.path.exists(os.path.join(data_path(), 'Music', 'MM.audiobin')): # Raise error. Maybe just skip and log a warning? raise FileNotFoundError(".MMRS sequence found but missing MM.audiobin") - + for f in zip.namelist(): if f.lower().endswith(".zsound"): split: str = f.split('.zsound') @@ -113,7 +113,7 @@ def process_sequence_mmrs(filepath: str, file_name: str, seq_type: str, include_ seq.zsounds.append(zsound) return seq return None - + def process_sequence_ootrs(filepath: str, file_name: str, seq_type: str, include_custom_audiobanks: bool, groups) -> Sequence: with zipfile.ZipFile(filepath) as zip: # Make sure meta file and seq file exists @@ -143,11 +143,11 @@ def process_sequence_ootrs(filepath: str, file_name: str, seq_type: str, include raise FileNotFoundError(f'No .seq file in: "{file_name}". This should never happen') if zbank_file and not bankmeta_file: raise FileNotFoundError(f'Custom track "{file_name}" contains .zbank but no .bankmeta') - + instrument_set = '-' cosmetic_name = filepath type_match = False - + # Read meta info with zip.open(meta_file, 'r') as stream: lines = io.TextIOWrapper(stream).readlines() # Use TextIOWrapper in order to get text instead of binary from the seq. diff --git a/Sequence.py b/Sequence.py index 95af48992..2f2226931 100644 --- a/Sequence.py +++ b/Sequence.py @@ -39,4 +39,4 @@ def copy(self) -> Sequence: copy.zbank_file = self.zbank_file copy.bankmeta = self.bankmeta copy.game = self.game - return copy \ No newline at end of file + return copy diff --git a/Unittest.py b/Unittest.py index 74e18be1f..bff042f88 100644 --- a/Unittest.py +++ b/Unittest.py @@ -823,7 +823,7 @@ def test_presets(self): self.verify_woth(spoiler) self.verify_playthrough(spoiler) self.verify_disables(spoiler) - + # remove this to run the fuzzer @unittest.skip("generally slow and failures can be ignored") def test_fuzzer(self): @@ -875,7 +875,7 @@ def test_audiobank(self): if not os.path.isfile('./ZOOTDEC.z64'): self.skipTest("Base ROM file not available.") - + rom: Rom = Rom("ZOOTDEC.z64") audiobank_file = rom.read_bytes(AUDIOBANK_ADDR, 0x1CA50) audiotable_index = rom.read_bytes(AUDIOTABLE_INDEX_ADDR, 0x80) # Read audiotable index into bytearray From 00e7bd00c7743c87db505e00a784b2a38baaa44d Mon Sep 17 00:00:00 2001 From: Rob Realmuto Date: Sat, 10 Feb 2024 02:02:36 -0500 Subject: [PATCH 08/10] Unsupported python support --- Audiobank.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Audiobank.py b/Audiobank.py index 34790deee..0f2e441fc 100644 --- a/Audiobank.py +++ b/Audiobank.py @@ -1,3 +1,4 @@ +from __future__ import annotations from io import FileIO from Rom import Rom From 35c60e9747852e8d63c2b91e4a392f35e59786fe Mon Sep 17 00:00:00 2001 From: Rob Realmuto Date: Fri, 29 Mar 2024 15:51:47 -0400 Subject: [PATCH 09/10] Test fixing web patcher --- Music.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Music.py b/Music.py index 9467458b5..fbca86b12 100644 --- a/Music.py +++ b/Music.py @@ -764,7 +764,7 @@ def randomize_music(rom: Rom, settings: Settings, log: CosmeticsLog, symbols: di sequences = {name: seq for name, seq in sequences.items() if name not in bgm_ids or name in music_mapping.values()} if available_sequences: for sequence_name in available_sequences.get("bgm", []): - sequences[sequence_name] = Sequence(sequence_name, sequence_name) + sequences[sequence_name] = Sequence(sequence_name, sequence_name, 'bgm') if settings.fanfares in ['random', 'random_custom_only'] or ff_mapped or ocarina_mapped: fanfare_sequences, target_fanfare_sequences, fanfare_groups = process_sequences(rom, ff_ids.values(), 'fanfare', disabled_source_sequences, disabled_target_sequences, custom_sequences_enabled, include_custom_audiobanks=custom_audiobanks_enabled, log=log) @@ -772,7 +772,7 @@ def randomize_music(rom: Rom, settings: Settings, log: CosmeticsLog, symbols: di fanfare_sequences = {name: seq for name, seq in fanfare_sequences.items() if name not in ff_ids or name in music_mapping.values()} if available_sequences: for sequence_name in available_sequences.get("fanfare", []): - fanfare_sequences[sequence_name] = Sequence(sequence_name, sequence_name) + fanfare_sequences[sequence_name] = Sequence(sequence_name, sequence_name, 'fanfare') # Handle groups. plando_groups = {n: s for n, s in log.src_dict.get('bgm_groups', {}).get('groups', {}).items()} From 08b17664f1ad95053a9f0381f9946060caff6c7c Mon Sep 17 00:00:00 2001 From: Rob Realmuto Date: Wed, 21 Aug 2024 20:01:21 -0400 Subject: [PATCH 10/10] Really MMR? You can't even keep your file extensions consistent? Why are we even trying to support this garbage --- MusicHelpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MusicHelpers.py b/MusicHelpers.py index 8b85e1248..f3246fe7a 100644 --- a/MusicHelpers.py +++ b/MusicHelpers.py @@ -49,7 +49,7 @@ def process_sequence_mmrs(filepath: str, file_name: str, seq_type: str, include_ #if f.endswith(".meta"): # meta_file = f # continue - if f.endswith(".zseq"): + if f.endswith(".zseq") or f.endswith(".seq"): seq_file = f continue if f.endswith(".zbank"):