2.3.0

CHANGELOG.md

@@ -7,6 +7,45 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [2.3.0] - 2023-11-05
+
+### Added
+
+- Support for parsing clang lld's map fles.
+- New functions:
+  - `MapFile.parseMapContents`/`MapFile::parse_map_contents`
+    - Parses the map contents passed as the argument, without requiring the map
+      being on an actual file.
+    - The map format will be guessed on the contents. Currently both the GNU ld
+      and clang ld.lld map formats are recognized.
+  - `MapFile.parseMapContentsGNU`/`MapFile::parse_map_contents_gnu`
+    - Parses the map contents passed as the argument, without requiring the map
+      being on an actual file.
+    - This function only parses the GNU ld map format.
+  - `MapFile.parseMapContentsLLD`/`MapFile::parse_map_contents_lld`
+    - Parses the map contents passed as the argument, without requiring the map
+      being on an actual file.
+    - This function only parses the clang ld.lld map format.
+- New members:
+  - `Symbol.align`/`Symbol::align`, `File.align`/`File::align` and
+    `Segment.align`/`Segment::align`: The alignment the given type. This member
+    will be filled by the parser only if the mapfile provides this information.
+
+### Changed
+
+- `MapFile.readMapFile`/`MapFile::read_map_file` can now guess the map format
+  between any of the known formats.
+- Some known symbol names will be automatically filtered out during the parsing
+  step.
+  - Currently only `gcc2_compiled.` is filtered out.
+
+### Fixed
+
+- Fix parser not detecting `*fill*` lines on GNU ld maps if they specified the
+  value that was used for filling/padding.
+- `.sbss`, `COMMON` and `.scommon` sections are now properly considered noload
+  sections.
+
 ## [2.2.1] - 2023-10-08
 
 ### Fixed
@@ -238,6 +277,7 @@ Full changes: <https://github.com/Decompollaborate/mapfile_parser/compare/702a73
 - Initial release
 
 [unreleased]: https://github.com/Decompollaborate/mapfile_parser/compare/master...develop
+[2.3.0]: https://github.com/Decompollaborate/mapfile_parser/compare/2.2.1...2.3.0
 [2.2.1]: https://github.com/Decompollaborate/mapfile_parser/compare/2.2.0...2.2.1
 [2.2.0]: https://github.com/Decompollaborate/mapfile_parser/compare/2.1.5...2.2.0
 [2.1.5]: https://github.com/Decompollaborate/mapfile_parser/compare/2.1.4...2.1.5

Cargo.toml

@@ -3,7 +3,7 @@
 
 [package]
 name = "mapfile_parser"
-version = "2.2.1"
+version = "2.3.0"
 edition = "2021"
 authors = ["Anghelo Carvajal <[email protected]>"]
 description = "Map file parser library focusing decompilation projects"
@@ -23,4 +23,10 @@ crate-type = ["cdylib", "staticlib", "rlib"]
 
 [dependencies]
 regex = "1.9.5"
-pyo3 = "0.19.0"
+# pyo3 = { version = "0.19.0", optional = true }
+pyo3 = { version = "0.19.0", optional = false }
+lazy_static = "1.4.0"
+
+[features]
+python_bindings = []
+# python_bindings = ["dep:pyo3"]

README.md

@@ -11,6 +11,14 @@ Map file parser library focusing decompilation projects.
 
 This library is available for Python3 and Rust
 
+## Features
+
+- Fast parsing written in Rust.
+- Support map formats:
+  - GNU ld
+  - clang lld
+- Built-in cli utilities to process the parsed map file (see [Examples](#examples)).
+
 ## Installing
 
 ### Python version
@@ -27,7 +35,7 @@ If you use a `requirements.txt` file in your repository, then you can add
 this library with the following line:
 
 ```txt
-mapfile_parser>=2.2.0,<3.0.0
+mapfile_parser>=2.3.0,<3.0.0
 ```
 
 #### Development version
@@ -57,9 +65,28 @@ you are doing. Proceed at your own risk.
 
 See this crate at <https://crates.io/crates/mapfile_parser>.
 
+To add this library to your project using Cargo:
+
+```bash
+cargo add mapfile_parser
+```
+
+Or add the following line manually to your `Cargo.toml` file:
+
 ```toml
-[build]
-mapfile_parser = "2.2.0"
+mapfile_parser = "2.3.0"
+```
+
+#### System-wide dependencies
+
+Due to how intrusive `pyo3` is (and even impossible to disable via Cargo
+features), Python development files are required to build and use the Rust
+version of this library
+
+To install those dependencies on a Debian/Ubuntu Linux distro:
+
+```bash
+sudo apt install libpython3-dev
 ```
 
 ## Versioning and changelog

pyproject.toml

@@ -3,7 +3,7 @@
 
 [project]
 name = "mapfile_parser"
-version = "2.2.1"
+version = "2.3.0"
 description = "Map file parser library focusing decompilation projects"
 readme = "README.md"
 requires-python = ">=3.7"
@@ -28,6 +28,6 @@ build-backend = "maturin"
 skip = ["cp36-*"]
 
 [tool.maturin]
-features = ["pyo3/extension-module"]
+features = ["pyo3/extension-module", "python_bindings"]
 # https://github.com/PyO3/maturin/blob/0dee40510083c03607834c821eea76964140a126/Readme.md#mixed-rustpython-projects
 python-source = "src"

src/mapfile_parser/__init__.py

@@ -5,7 +5,7 @@
 
 from __future__ import annotations
 
-__version_info__ = (2, 2, 1)
+__version_info__ = (2, 3, 0)
 __version__ = ".".join(map(str, __version_info__))
 __author__ = "Decompollaborate"
 

src/mapfile_parser/mapfile.py

@@ -13,6 +13,7 @@
 from .progress_stats import ProgressStats
 from . import utils
 
+from .mapfile_rs import MapFile as MapFileRs
 
 regex_fileDataEntry = re.compile(r"^\s+(?P<section>\.[^\s]+)\s+(?P<vram>0x[^\s]+)\s+(?P<size>0x[^\s]+)\s+(?P<name>[^\s]+)$")
 regex_functionEntry = re.compile(r"^\s+(?P<vram>0x[^\s]+)\s+(?P<name>[^\s]+)$")
@@ -64,6 +65,7 @@ class Symbol:
     vram: int
     size: int|None = None # in bytes
     vrom: int|None = None
+    align: int|None = None
 
     def getVramStr(self) -> str:
         return f"0x{self.vram:08X}"
@@ -140,6 +142,7 @@ class File:
     size: int # in bytes
     sectionType: str
     vrom: int|None = None
+    align: int|None = None
     _symbols: list[Symbol] = dataclasses.field(default_factory=list)
 
     @property
@@ -307,6 +310,7 @@ class Segment:
     vram: int
     size: int
     vrom: int
+    align: int|None = None
     _filesList: list[File] = dataclasses.field(default_factory=list)
 
     def serializeVram(self, humanReadable: bool=True) -> str|int|None:
@@ -478,23 +482,76 @@ def __init__(self):
         #! @deprecated
         self.debugging: bool = False
 
-    def readMapFile(self, mapPath: Path):
-        from .mapfile_rs import MapFile
-
-        nativeMapFile = MapFile()
-        nativeMapFile.readMapFile(mapPath)
-
+    def _transferContentsFromNativeMapFile(self, nativeMapFile: MapFileRs):
         for segment in nativeMapFile:
-            newSegment = Segment(segment.name, segment.vram, segment.size, segment.vrom)
+            newSegment = Segment(segment.name, segment.vram, segment.size, segment.vrom, segment.align)
             for file in segment:
-                newFile = File(file.filepath, file.vram, file.size, file.sectionType, file.vrom)
+                newFile = File(file.filepath, file.vram, file.size, file.sectionType, file.vrom, file.align)
                 for symbol in file:
-                    newSymbol = Symbol(symbol.name, symbol.vram, symbol.size, symbol.vrom)
+                    newSymbol = Symbol(symbol.name, symbol.vram, symbol.size, symbol.vrom, symbol.align)
 
                     newFile._symbols.append(newSymbol)
                 newSegment._filesList.append(newFile)
             self._segmentsList.append(newSegment)
 
+    def readMapFile(self, mapPath: Path):
+        """
+        Opens the mapfile pointed by the `mapPath` argument and parses it.
+
+        The format of the map will be guessed based on its contents.
+
+        Currently supported map formats:
+        - GNU ld
+        - clang ld.lld
+        """
+
+        nativeMapFile = MapFileRs()
+        nativeMapFile.readMapFile(mapPath)
+
+        self._transferContentsFromNativeMapFile(nativeMapFile)
+
+    def parseMapContents(self, mapContents: str):
+        """
+        Parses the contents of the map.
+
+        The `mapContents` argument must contain the contents of a mapfile.
+
+        The format of the map will be guessed based on its contents.
+
+        Currently supported mapfile formats:
+        - GNU ld
+        - clang ld.lld
+        """
+
+        nativeMapFile = MapFileRs()
+        nativeMapFile.parseMapContents(mapContents)
+
+        self._transferContentsFromNativeMapFile(nativeMapFile)
+
+    def parseMapContentsGNU(self, mapContents: str):
+        """
+        Parses the contents of a GNU ld map.
+
+        The `mapContents` argument must contain the contents of a GNU ld mapfile.
+        """
+
+        nativeMapFile = MapFileRs()
+        nativeMapFile.parseMapContentsGNU(mapContents)
+
+        self._transferContentsFromNativeMapFile(nativeMapFile)
+
+    def parseMapContentsLLD(self, mapContents: str):
+        """
+        Parses the contents of a clang ld.lld map.
+
+        The `mapContents` argument must contain the contents of a clang ld.lld mapfile.
+        """
+
+        nativeMapFile = MapFileRs()
+        nativeMapFile.parseMapContentsLLD(mapContents)
+
+        self._transferContentsFromNativeMapFile(nativeMapFile)
+
 
     def filterBySectionType(self, sectionType: str) -> MapFile:
         newMapFile = MapFile()

src/mapfile_parser/mapfile_parser.pyi

@@ -60,8 +60,9 @@ class Symbol:
     vram: int
     size: int|None # in bytes
     vrom: int|None
+    align: int|None
 
-    def __init__(self, name: str, vram: int, size: int|None=None, vrom: int|None=None): ...
+    def __init__(self, name: str, vram: int, size: int|None=None, vrom: int|None=None, align: int|None=None): ...
 
     def getVramStr(self) -> str: ...
     def getSizeStr(self) -> str: ...
@@ -92,9 +93,10 @@ class File:
     size: int # in bytes
     sectionType: str
     vrom: int|None
+    align: int|None
     # _symbols: list[Symbol]
 
-    def __init__(self, filepath: Path, vram: int, size: int, section_type: str, vrom: int|None=None) -> None: ...
+    def __init__(self, filepath: Path, vram: int, size: int, section_type: str, vrom: int|None=None, align: int|None=None) -> None: ...
 
     @property
     def filepath(self) -> Path: ...
@@ -147,9 +149,10 @@ class Segment:
     vram: int
     size: int
     vrom: int
+    align: int|None
     # _filesList: list[File] = dataclasses.field(default_factory=list)
 
-    def __init__(self, name: str, vram: int, size: int, vrom: int): ...
+    def __init__(self, name: str, vram: int, size: int, vrom: int, align: int|None=None): ...
 
     def serializeVram(self, humanReadable: bool=True) -> str|int|None: ...
     def serializeSize(self, humanReadable: bool=True) -> str|int|None: ...
@@ -192,6 +195,9 @@ class MapFile:
     def __init__(self) -> None: ...
 
     def readMapFile(self, mapPath: Path) -> None: ...
+    def parseMapContents(self, mapContents: str) -> None: ...
+    def parseMapContentsGNU(self, mapContents: str) -> None: ...
+    def parseMapContentsLLD(self, mapContents: str) -> None: ...
 
     def filterBySectionType(self, sectionType: str) -> MapFile: ...
     def getEveryFileExceptSectionType(self, sectionType: str) -> MapFile: ...