Skip to content

Commit

Permalink
Merge pull request #32 from Decompollaborate/develop
Browse files Browse the repository at this point in the history
2.7.0
  • Loading branch information
AngheloAlf authored Sep 25, 2024
2 parents 85197c5 + fe97c35 commit 5f57c95
Show file tree
Hide file tree
Showing 20 changed files with 648,840 additions and 23 deletions.
24 changes: 24 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

## [2.7.0] - 2024-09-24

### Added

- `MapFile.findSymbolByVram` and `MapFile.findSymbolByVrom` methods.
- Allow to search a symbol given a given address. The address will be treated
as either a vram address or a vrom address respectively.
- Add `--vram`, `--vrom` and `--name` arguments to `sym_info` frontend.
- Allow to tell to `sym_info` exactly how to treat the argument instead of
trying to guess how to use it.
- `sym_info` can now detect that an address may belong to a file even when the
symbol itself may not exist on the mapfile.
- This can happen for local symbols, for example for rodata literals.

### Deprecated

- `MapFile.findSymbolByVramOrVrom`.
- Use `MapFile.findSymbolByVram` and `MapFile.findSymbolByVrom` instead.

### Fixed

- Fix typo that prevented using `jsonify`.

## [2.6.0] - 2024-08-26

### Added
Expand Down Expand Up @@ -392,6 +415,7 @@ Full changes: <https://github.com/Decompollaborate/mapfile_parser/compare/702a73
- Initial release

[unreleased]: https://github.com/Decompollaborate/mapfile_parser/compare/master...develop
[2.7.0]: https://github.com/Decompollaborate/mapfile_parser/compare/2.6.0...2.7.0
[2.6.0]: https://github.com/Decompollaborate/mapfile_parser/compare/2.5.1...2.6.0
[2.5.1]: https://github.com/Decompollaborate/mapfile_parser/compare/2.5.0...2.5.1
[2.5.0]: https://github.com/Decompollaborate/mapfile_parser/compare/2.4.0...2.5.0
Expand Down
2 changes: 1 addition & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

[package]
name = "mapfile_parser"
version = "2.6.0"
version = "2.7.0"
edition = "2021"
rust-version = "1.65.0"
authors = ["Anghelo Carvajal <[email protected]>"]
Expand Down
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ If you use a `requirements.txt` file in your repository, then you can add
this library with the following line:

```txt
mapfile_parser>=2.6.0,<3.0.0
mapfile_parser>=2.7.0,<3.0.0
```

#### Development version
Expand Down Expand Up @@ -74,7 +74,7 @@ cargo add mapfile_parser
Or add the following line manually to your `Cargo.toml` file:

```toml
mapfile_parser = "2.6.0"
mapfile_parser = "2.7.0"
```

## Versioning and changelog
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

[project]
name = "mapfile_parser"
version = "2.6.0"
version = "2.7.0"
description = "Map file parser library focusing decompilation projects"
readme = "README.md"
requires-python = ">=3.8"
Expand Down
2 changes: 1 addition & 1 deletion src/mapfile_parser/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

from __future__ import annotations

__version_info__ = (2, 6, 0)
__version_info__ = (2, 7, 0)
__version__ = ".".join(map(str, __version_info__))# + "-dev0"
__author__ = "Decompollaborate"

Expand Down
21 changes: 16 additions & 5 deletions src/mapfile_parser/frontends/first_diff.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,10 +61,12 @@ def doFirstDiff(mapPath: Path, expectedMapPath: Path, romPath: Path, expectedRom
or builtRom[i + 3] != expectedRom[i + 3]
):
if diffs == 0:
vromInfo = builtMapFile.findSymbolByVramOrVrom(i)
vromInfo, possibleFiles = builtMapFile.findSymbolByVrom(i)
extraMessage = ""
if vromInfo is not None:
extraMessage = f", {vromInfo.getAsStrPlusOffset()}"
elif len(possibleFiles) > 0:
extraMessage = f", in file {possibleFiles[0].asStr()}"
print(f"First difference at ROM addr 0x{i:X}{extraMessage}")
builtBytes = builtRom[i : i + 4]
expectedBytes = expectedRom[i : i + 4]
Expand All @@ -80,15 +82,13 @@ def doFirstDiff(mapPath: Path, expectedMapPath: Path, romPath: Path, expectedRom
len(map_search_diff) < diffCount
and builtRom[i+endian_diff] >> 2 != expectedRom[i+endian_diff] >> 2
):
vromInfo = builtMapFile.findSymbolByVramOrVrom(i)
vromInfo, possibleFiles = builtMapFile.findSymbolByVrom(i)
if vromInfo is not None:
vromMessage = vromInfo.getAsStr()
if vromMessage not in map_search_diff:
map_search_diff.add(vromMessage)

extraMessage = ""
if vromInfo is not None:
extraMessage = f", {vromInfo.getAsStrPlusOffset()}"
extraMessage = f", {vromInfo.getAsStrPlusOffset()}"
print(f"Instruction difference at ROM addr 0x{i:X}{extraMessage}")
builtBytes = builtRom[i : i + 4]
expectedBytes = expectedRom[i : i + 4]
Expand All @@ -98,6 +98,17 @@ def doFirstDiff(mapPath: Path, expectedMapPath: Path, romPath: Path, expectedRom
expectedConverted = bytesConverterCallback(expectedBytes, expectedMapFile)
if builtConverted is not None and expectedConverted is not None:
print(f"{builtConverted} vs {expectedConverted}")
elif len(possibleFiles) > 0:
extraMessage = f", in file {possibleFiles[0].asStr()}"
print(f"Instruction difference at ROM addr 0x{i:X}{extraMessage}")
builtBytes = builtRom[i : i + 4]
expectedBytes = expectedRom[i : i + 4]
print(f"Bytes: {utils.hexbytes(builtBytes, addColons=addColons)} vs {utils.hexbytes(expectedBytes, addColons=addColons)}")
if bytesConverterCallback is not None:
builtConverted = bytesConverterCallback(builtBytes, builtMapFile)
expectedConverted = bytesConverterCallback(expectedBytes, expectedMapFile)
if builtConverted is not None and expectedConverted is not None:
print(f"{builtConverted} vs {expectedConverted}")

if len(map_search_diff) >= diffCount and diffs > shift_cap:
break
Expand Down
2 changes: 1 addition & 1 deletion src/mapfile_parser/frontends/jsonify.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ def processArguments(args: argparse.Namespace):
mapPath: Path = args.mapfile
outputPath: Path|None = Path(args.output) if args.output is not None else None
machine: bool = args.machine
applyFixes: bool = args.applyFixes
applyFixes: bool = args.apply_fixes

exit(doJsonify(mapPath, outputPath, humanReadable=not machine, applyFixes=applyFixes))

Expand Down
50 changes: 39 additions & 11 deletions src/mapfile_parser/frontends/sym_info.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,36 +12,64 @@
from .. import utils


def doSymInfo(mapPath: Path, symName: str) -> int:
def doSymInfo(mapPath: Path, symName: str, *, as_vram: bool=False, as_vrom: bool=False, as_name: bool=False) -> int:
if not mapPath.exists():
print(f"Could not find mapfile at '{mapPath}'")
return 1

mapFile = mapfile.MapFile()
mapFile.readMapFile(mapPath)

if utils.convertibleToInt(symName, 0):
info = mapFile.findSymbolByVramOrVrom(int(symName, 0))
possibleFiles: list[mapfile.File] = []

if as_vram:
address = int(symName, 0)
info, possibleFiles = mapFile.findSymbolByVram(address)
elif as_vrom:
address = int(symName, 0)
info, possibleFiles = mapFile.findSymbolByVrom(address)
elif as_name:
info = mapFile.findSymbolByName(symName)

# Start the guessing game
elif utils.convertibleToInt(symName, 0):
address = int(symName, 0)
info, possibleFiles = mapFile.findSymbolByVram(address)
if info is None:
info, possibleFiles2 = mapFile.findSymbolByVrom(address)
possibleFiles.extend(possibleFiles2)
else:
info = mapFile.findSymbolByName(symName)

if info is None:
print(f"'{symName}' not found in map file '{mapPath}'")
return 1
print(info.getAsStrPlusOffset(symName))
return 0
if info is not None:
print(info.getAsStrPlusOffset(symName))
return 0
print(f"'{symName}' not found in map file '{mapPath}'")
if len(possibleFiles) > 0:
print("But it may be a local symbol of either of the following files:")
for f in possibleFiles:
print(f" {f.asStr()})")
return 1


def processArguments(args: argparse.Namespace):
mapPath: Path = args.mapfile
symName: str = args.symname
as_vram: bool = args.vram
as_vrom: bool = args.vrom
as_name: bool = args.name

exit(doSymInfo(mapPath, symName))
exit(doSymInfo(mapPath, symName, as_vram=as_vram, as_vrom=as_vrom, as_name=as_name))

def addSubparser(subparser: argparse._SubParsersAction[argparse.ArgumentParser]):
parser = subparser.add_parser("sym_info", help="Display various information about a symbol or address.")

parser.add_argument("mapfile", help="Path to a map file", type=Path)
parser.add_argument("symname", help="symbol name or VROM/VRAM address to lookup")
parser.add_argument("mapfile", help="Path to a map file.", type=Path)
parser.add_argument("symname", help="Symbol name or VROM/VRAM address to lookup. How to treat this argument will be guessed.")

vram_vrom_group = parser.add_mutually_exclusive_group()
vram_vrom_group.add_argument("--vram", help="Treat the argument as a VRAM address instead of guessing.", action="store_true")
vram_vrom_group.add_argument("--vrom", help="Treat the argument as a VROM address instead of guessing.", action="store_true")
vram_vrom_group.add_argument("--name", help="Treat the argument as a symbol name instead of guessing.", action="store_true")

parser.set_defaults(func=processArguments)
119 changes: 119 additions & 0 deletions src/mapfile_parser/mapfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,7 @@ def findSymbolByName(self, symName: str) -> Symbol|None:
return sym
return None

#! @deprecated: Use either `findSymbolByVram` or `findSymbolByVrom` instead.
def findSymbolByVramOrVrom(self, address: int) -> tuple[Symbol, int]|None:
prevVram = self.vram
prevVrom = self.vrom
Expand Down Expand Up @@ -244,6 +245,59 @@ def findSymbolByVramOrVrom(self, address: int) -> tuple[Symbol, int]|None:

return None

def findSymbolByVram(self, address: int) -> tuple[Symbol, int]|None:
prevSym: Symbol|None = None

for sym in self._symbols:
if sym.vram == address:
return sym, 0

if prevSym is not None:
if sym.vram > address:
offset = address - prevSym.vram
if offset < 0:
return None
return prevSym, offset

prevSym = sym

if prevSym is not None:
if prevSym.size is not None and prevSym.vram + prevSym.size > address:
offset = address - prevSym.vram
if offset < 0:
return None
return prevSym, offset

return None

def findSymbolByVrom(self, address: int) -> tuple[Symbol, int]|None:
prevVrom = self.vrom if self.vrom is not None else 0
prevSym: Symbol|None = None

for sym in self._symbols:
if sym.vrom == address:
return sym, 0

if prevSym is not None:
if sym.vrom is not None and sym.vrom > address:
offset = address - prevVrom
if offset < 0:
return None
return prevSym, offset

if sym.vrom is not None:
prevVrom = sym.vrom
prevSym = sym

if prevSym is not None:
if prevSym.vrom is not None and prevSym.size is not None and prevSym.vrom + prevSym.size > address:
offset = address - prevVrom
if offset < 0:
return None
return prevSym, offset

return None


@staticmethod
def printCsvHeader(printVram: bool=True):
Expand Down Expand Up @@ -292,6 +346,9 @@ def toJson(self, humanReadable: bool=True) -> dict[str, Any]:
fileDict["symbols"] = symbolsList
return fileDict

def asStr(self) -> str:
return f"{self.filepath}({self.sectionType}) (VRAM: {self.serializeVram(True)}, VROM: {self.serializeVrom(True)}, SIZE: {self.serializeSize(humanReadable=True)})"


def copySymbolList(self) -> list[Symbol]:
"""Returns a copy (not a reference) of the internal symbol list"""
Expand Down Expand Up @@ -385,6 +442,7 @@ def findSymbolByName(self, symName: str) -> FoundSymbolInfo|None:
return FoundSymbolInfo(file, sym)
return None

#! @deprecated: Use either `findSymbolByVram` or `findSymbolByVrom` instead.
def findSymbolByVramOrVrom(self, address: int) -> FoundSymbolInfo|None:
for file in self._filesList:
pair = file.findSymbolByVramOrVrom(address)
Expand All @@ -393,6 +451,30 @@ def findSymbolByVramOrVrom(self, address: int) -> FoundSymbolInfo|None:
return FoundSymbolInfo(file, sym, offset)
return None

def findSymbolByVram(self, address: int) -> tuple[FoundSymbolInfo|None, list[File]]:
possibleFiles: list[File] = []
for file in self._filesList:
pair = file.findSymbolByVram(address)
if pair is not None:
sym, offset = pair
return FoundSymbolInfo(file, sym, offset), []
if address >= file.vram and address < file.vram + file.size:
possibleFiles.append(file)
return None, possibleFiles

def findSymbolByVrom(self, address: int) -> tuple[FoundSymbolInfo|None, list[File]]:
possibleFiles: list[File] = []
for file in self._filesList:
if file.vrom is None:
continue
pair = file.findSymbolByVrom(address)
if pair is not None:
sym, offset = pair
return FoundSymbolInfo(file, sym, offset), []
if address >= file.vrom and address < file.vrom + file.size:
possibleFiles.append(file)
return None, possibleFiles


def mixFolders(self) -> Segment:
newSegment = Segment(self.name, self.vram, self.size, self.vrom)
Expand Down Expand Up @@ -622,13 +704,50 @@ def findSymbolByName(self, symName: str) -> FoundSymbolInfo|None:
return info
return None

#! @deprecated: Use either `findSymbolByVram` or `findSymbolByVrom` instead.
def findSymbolByVramOrVrom(self, address: int) -> FoundSymbolInfo|None:
for segment in self._segmentsList:
info = segment.findSymbolByVramOrVrom(address)
if info is not None:
return info
return None

def findSymbolByVram(self, address: int) -> tuple[FoundSymbolInfo|None, list[File]]:
"""
Returns a symbol with the specified VRAM address (or with an addend) if
it exists on the mapfile.
If no symbol if found, then a list of possible files where this symbol
may belong to is returned. This may happen if the symbol is not
globally visible.
"""

possibleFiles: list[File] = []
for segment in self._segmentsList:
info, possibleFilesAux = segment.findSymbolByVram(address)
if info is not None:
return info, []
possibleFiles.extend(possibleFilesAux)
return None, possibleFiles

def findSymbolByVrom(self, address: int) -> tuple[FoundSymbolInfo|None, list[File]]:
"""
Returns a symbol with the specified VRAM address (or with an addend) if
it exists on the mapfile.
If no symbol if found, then a list of possible files where this symbol
may belong to is returned. This may happen if the symbol is not
globally visible.
"""

possibleFiles: list[File] = []
for segment in self._segmentsList:
info, possibleFilesAux = segment.findSymbolByVrom(address)
if info is not None:
return info, []
possibleFiles.extend(possibleFilesAux)
return None, possibleFiles

def findLowestDifferingSymbol(self, otherMapFile: MapFile) -> tuple[Symbol, File, Symbol|None]|None:
minVram = None
found = None
Expand Down
Loading

0 comments on commit 5f57c95

Please sign in to comment.