Add test to check progress by checking symbol names

.github/workflows/tests.yml

@@ -4,12 +4,13 @@ name: Tests cases
 on: [push, pull_request]
 
 jobs:
-  tests_cases:
-    name: Tests cases
+  check_if_output_files_changed:
+    name: Check if output files changed
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v4
+    - name: Checkout reposistory
+      uses: actions/checkout@v4
 
     - name: Setup Rust toolchain
       uses: actions-rs/toolchain@v1
@@ -38,3 +39,28 @@ jobs:
         echo "Changed files: ${{ steps.tests_changes.outputs.changed_files }}"
         echo "Please install the latest changes, run \`python3 tests/update_outputs.py\`, check the changes are desirable and commit the result"
         exit 1
+
+  check_progress_nonmatchings:
+    name: Check progress by NON_MATCHING symbols
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout reposistory
+      uses: actions/checkout@v4
+
+    - name: Setup Rust toolchain
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        override: true
+
+    - name: Setup requirements
+      run: |
+        python3 -m pip install -U -r requirements.txt
+        python3 -m pip install -U maturin
+
+    - name: Install local mapfile_parser
+      run: python3 -m pip install .
+
+    - name: Update tests outputs
+      run: python3 tests/check_progress_nonmatchings.py

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Add a few utility methods to `ProgressStats`.
+
 ### Changed
 
 - `pyo3` is no longer needed to use this crate as Rust-only library.

src/mapfile_parser/mapfile_parser.pyi

@@ -47,11 +47,30 @@ class ProgressStats:
     @property
     def total(self) -> int: ...
 
+    def undecompedPercentage(self) -> float:
+        ...
+
+    def decompedPercentage(self) -> float:
+        ...
+
+    def undecompedPercentageTotal(self, totalStats: ProgressStats) -> float:
+        ...
+
+    def decompedPercentageTotal(self, totalStats: ProgressStats) -> float:
+        ...
+
     def getAsFrogressEntry(self, name: str) -> dict[str, int]: ...
 
+    @staticmethod
+    def getHeaderAsStr() -> str:
+        ...
+
     @staticmethod
     def printHeader() -> None: ...
 
+    def getEntryAsStr(self, category: str, totalStats: ProgressStats) -> str:
+        ...
+
     def print(self, category: str, totalStats: ProgressStats) -> None: ...
 
 

src/mapfile_parser/progress_stats.py

@@ -17,18 +17,37 @@ class ProgressStats:
     def total(self) -> int:
         return self.undecompedSize + self.decompedSize
 
+    def undecompedPercentage(self) -> float:
+        return self.undecompedSize / self.total * 100
+
+    def decompedPercentage(self) -> float:
+        return self.decompedSize / self.total * 100
+
+    def undecompedPercentageTotal(self, totalStats: ProgressStats) -> float:
+        return self.undecompedSize / totalStats.total * 100
+
+    def decompedPercentageTotal(self, totalStats: ProgressStats) -> float:
+        return self.decompedSize / totalStats.total * 100
+
     def getAsFrogressEntry(self, name: str) -> dict[str, int]:
         categories: dict[str, int] = {}
         categories[name] = self.decompedSize
         categories[f"{name}/total"] = self.total
         return categories
 
+    @staticmethod
+    def getHeaderAsStr() -> str:
+        return f"{'Category':<28}: {'DecompedSize':>12} / {'Total':>8} {'OfFolder':>10}%  ({'OfTotal':>20}%)"
+
     @staticmethod
     def printHeader():
-        print(f"{'Category':<28}: {'DecompedSize':>12} / {'Total':>8} {'OfFolder':>10}%  ({'OfTotal':>20}%)")
+        print(ProgressStats.getHeaderAsStr())
+
+    def getEntryAsStr(self, category: str, totalStats: ProgressStats) -> str:
+        return f"{category:<28}: {self.decompedSize:>12} / {self.total:>8} {self.decompedPercentage():>10.4f}%  ({self.decompedPercentageTotal(totalStats):>8.4f}% / {self.total / totalStats.total * 100:>8.4f}%)"
 
     def print(self, category: str, totalStats: ProgressStats):
-        print(f"{category:<28}: {self.decompedSize:>12} / {self.total:>8} {self.decompedSize / self.total * 100:>10.4f}%  ({self.decompedSize / totalStats.total * 100:>8.4f}% / {self.total / totalStats.total * 100:>8.4f}%)")
+        print(self.getEntryAsStr(category, totalStats))
 
 
 def printStats(totalStats: ProgressStats, progressPerFolder: dict[str, ProgressStats]):

src/rs/mapfile.rs

@@ -576,7 +576,7 @@ impl MapFile {
                         .join(extensionless_file_path.clone())
                         .join(func.name.clone() + ".s");
 
-                    let sym_size = func.size.unwrap_or(0) as u32;
+                    let sym_size = func.size.unwrap_or(0) as usize;
 
                     if whole_file_is_undecomped || func_asm_path.exists() {
                         total_stats.undecomped_size += sym_size;

src/rs/progress_stats.rs

@@ -9,9 +9,9 @@ use pyo3::prelude::*;
 #[derive(Debug, Clone)]
 #[cfg_attr(feature = "python_bindings", pyclass(module = "mapfile_parser"))]
 pub struct ProgressStats {
-    pub undecomped_size: u32,
+    pub undecomped_size: usize,
 
-    pub decomped_size: u32,
+    pub decomped_size: usize,
 }
 
 impl ProgressStats {
@@ -22,35 +22,65 @@ impl ProgressStats {
         }
     }
 
-    pub fn total(&self) -> u32 {
+    pub fn total(&self) -> usize {
         self.undecomped_size + self.decomped_size
     }
 
-    pub fn get_as_frogress_entry(&self, name: &str) -> HashMap<String, u32> {
-        let mut categories: HashMap<String, u32> = HashMap::new();
+    pub fn undecomped_percentage(&self) -> f32 {
+        self.undecomped_size as f32 / self.total() as f32 * 100.0
+    }
+
+    pub fn decomped_percentage(&self) -> f32 {
+        self.decomped_size as f32 / self.total() as f32 * 100.0
+    }
+
+    pub fn undecomped_percentage_total(&self, total_stats: &Self) -> f32 {
+        self.undecomped_size as f32 / total_stats.total() as f32 * 100.0
+    }
+
+    pub fn decomped_percentage_total(&self, total_stats: &Self) -> f32 {
+        self.decomped_size as f32 / total_stats.total() as f32 * 100.0
+    }
+
+    pub fn get_as_frogress_entry(&self, name: &str) -> HashMap<String, usize> {
+        let mut categories: HashMap<String, usize> = HashMap::new();
 
         categories.insert(name.to_string(), self.decomped_size);
         categories.insert(format!("{}/total", name), self.total());
 
         categories
     }
 
-    pub fn print_header() {
-        println!(
+    pub fn get_header_as_str() -> String {
+        format!(
             "{:<28}: {:>12} / {:>8} {:>10}%  ({:>20}%)",
             "Category", "DecompedSize", "Total", "OfFolder", "OfTotal"
-        );
+        )
     }
 
-    pub fn print(&self, category: &str, total_stats: &Self) {
+    pub fn print_header() {
         println!(
+            "{}",
+            Self::get_header_as_str()
+        );
+    }
+
+    pub fn get_entry_as_str(&self, category: &str, total_stats: &Self) -> String {
+        format!(
             "{:<28}: {:>12} / {:>8} {:>10.4}%  ({:>8.4}% / {:>8.4}%)",
             category,
             self.decomped_size,
             self.total(),
-            self.decomped_size as f32 / self.total() as f32 * 100.0,
-            self.decomped_size as f32 / total_stats.total() as f32 * 100.0,
+            self.decomped_percentage(),
+            self.decomped_percentage_total(total_stats),
             self.total() as f32 / total_stats.total() as f32 * 100.0
+        )
+    }
+
+    pub fn print(&self, category: &str, total_stats: &Self) {
+        println!(
+            "{}",
+            self.get_entry_as_str(category, total_stats)
         );
     }
 }
@@ -78,44 +108,69 @@ pub(crate) mod python_bindings {
         /* Getters and setters */
 
         #[getter]
-        fn get_undecompedSize(&self) -> PyResult<u32> {
+        fn get_undecompedSize(&self) -> PyResult<usize> {
             Ok(self.undecomped_size)
         }
 
         #[setter]
-        fn set_undecompedSize(&mut self, value: u32) -> PyResult<()> {
+        fn set_undecompedSize(&mut self, value: usize) -> PyResult<()> {
             self.undecomped_size = value;
             Ok(())
         }
 
         #[getter]
-        fn get_decompedSize(&self) -> PyResult<u32> {
+        fn get_decompedSize(&self) -> PyResult<usize> {
             Ok(self.decomped_size)
         }
 
         #[setter]
-        fn set_decompedSize(&mut self, value: u32) -> PyResult<()> {
+        fn set_decompedSize(&mut self, value: usize) -> PyResult<()> {
             self.decomped_size = value;
             Ok(())
         }
 
         #[getter]
         #[pyo3(name = "total")]
-        pub fn py_total(&self) -> u32 {
+        pub fn py_total(&self) -> usize {
             self.total()
         }
 
         /* Methods */
 
-        pub fn getAsFrogressEntry(&self, name: &str) -> HashMap<String, u32> {
+        fn undecompedPercentage(&self) -> f32 {
+            self.undecomped_percentage()
+        }
+
+        fn decompedPercentage(&self) -> f32 {
+            self.decomped_percentage()
+        }
+
+        fn undecompedPercentageTotal(&self, total_stats: &Self) -> f32 {
+            self.undecomped_percentage_total(total_stats)
+        }
+
+        fn decompedPercentageTotal(&self, total_stats: &Self) -> f32 {
+            self.decomped_percentage_total(total_stats)
+        }
+
+        pub fn getAsFrogressEntry(&self, name: &str) -> HashMap<String, usize> {
             self.get_as_frogress_entry(name)
         }
 
+        #[staticmethod]
+        pub fn getHeaderAsStr() -> String {
+            Self::get_header_as_str()
+        }
+
         #[staticmethod]
         pub fn printHeader() {
             Self::print_header()
         }
 
+        pub fn getEntryAsStr(&self, category: &str, total_stats: &Self) -> String {
+            self.get_entry_as_str(category, total_stats)
+        }
+
         #[pyo3(name = "print")]
         pub fn py_print(&self, category: &str, total_stats: &Self) {
             self.print(category, total_stats)

tests/check_progress_nonmatchings.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: © 2023 Decompollaborate
+# SPDX-License-Identifier: MIT
+
+from __future__ import annotations
+
+## This test checks if it is possible to modify a class's member
+## (important because of Rust/Python interoperability)
+
+import mapfile_parser
+from pathlib import Path
+
+def getProgressFromMapFile(mapFile: mapfile_parser.MapFile, aliases: dict[str, str]=dict(), pathIndex: int=2) -> tuple[mapfile_parser.ProgressStats, dict[str, mapfile_parser.ProgressStats]]:
+    totalStats = mapfile_parser.ProgressStats()
+    progressPerFolder: dict[str, mapfile_parser.ProgressStats] = dict()
+
+    for segment in mapFile:
+        for file in segment:
+            if len(file) == 0:
+                continue
+
+            folder = file.filepath.parts[pathIndex]
+            if folder in aliases:
+                folder = aliases[folder]
+
+            if folder not in progressPerFolder:
+                progressPerFolder[folder] = mapfile_parser.ProgressStats()
+
+            for func in file:
+                if func.name.endswith(".NON_MATCHING"):
+                    continue
+
+                funcNonMatching = f"{func.name}.NON_MATCHING"
+
+                funcSize = func.size
+                assert funcSize is not None
+
+                if mapFile.findSymbolByName(funcNonMatching) is not None:
+                    totalStats.undecompedSize += funcSize
+                    progressPerFolder[folder].undecompedSize += funcSize
+                else:
+                    totalStats.decompedSize += funcSize
+                    progressPerFolder[folder].decompedSize += funcSize
+
+    return totalStats, progressPerFolder
+
+def getProgress(mapPath: Path, version: str, pathIndex: int=2) -> tuple[mapfile_parser.ProgressStats, dict[str, mapfile_parser.ProgressStats]]:
+    mapFile = mapfile_parser.MapFile()
+    mapFile.readMapFile(mapPath)
+
+    for segment in mapFile:
+        for file in segment:
+            if len(file) == 0:
+                continue
+
+            filepathParts = list(file.filepath.parts)
+            if version in filepathParts:
+                filepathParts.remove(version)
+            file.filepath = Path(*filepathParts)
+
+            # Fix symbol size calculation because of NON_MATCHING symbols
+            for sym in file:
+                if sym.name.endswith(".NON_MATCHING") and sym.size != 0:
+                    realSym = file.findSymbolByName(sym.name.replace(".NON_MATCHING", ""))
+                    if realSym is not None and realSym.size == 0:
+                        realSym.size = sym.size
+                        sym.size = 0
+
+    return getProgressFromMapFile(mapFile.filterBySectionType(".text"), aliases={"ultralib": "libultra"}, pathIndex=pathIndex)
+
+
+cases: list[tuple[Path, str, mapfile_parser.ProgressStats]] = [
+    (Path("tests/maps/drmario64.cn.map"), "cn", mapfile_parser.ProgressStats(undecompedSize=273028, decompedSize=199196)),
+    (Path("tests/maps/drmario64.us.lld.map"), "us", mapfile_parser.ProgressStats(undecompedSize=170720, decompedSize=272764)),
+    (Path("tests/maps/drmario64.us.map"), "us", mapfile_parser.ProgressStats(undecompedSize=170720, decompedSize=272128)),
+    (Path("tests/maps/puzzleleague64.usa.map"), "usa", mapfile_parser.ProgressStats(undecompedSize=263668, decompedSize=454604)),
+]
+
+
+errors = 0
+for (mapPath, version, expected) in cases:
+    print(mapPath)
+
+    totalStats, progressPerFolder = getProgress(mapPath, version)
+
+    print(f"    {expected} {expected.decompedPercentage():>10.4f}%")
+    print(f"    {totalStats} {totalStats.decompedPercentage():>10.4f}%")
+
+    if totalStats == expected:
+        print("        Ok")
+    else:
+        print("        Wrong")
+        errors += 1
+
+    print()
+
+exit(errors)