Debug tools for loading SSZ object from file

docs/docs/load-ssz.md

@@ -0,0 +1,88 @@
+# Load SSZ file into Pyspec SSZ object
+
+## Common file formats
+
+### Beacon APIs response
+
+The [Beacon APIs](https://github.com/ethereum/beacon-APIs) return JSON format response. In case of a successful response, the requested object can be found within the `response['data']` field.
+
+#### Helpers
+
+##### `eth2spec.debug.tools.get_ssz_object_from_json_file`
+
+```python
+get_ssz_object_from_json_file(container: Container, file_path: str) -> SSZObject
+```
+
+Get the `SSZObject` from a specific JSON file.
+
+###### Arguments
+- `container: Container`: the SSZ Container class. e.g., `BeaconState`.
+- `file_path: str`: the path of the JSON file.
+
+###### Example
+
+```python
+import eth2spec.capella.mainnet as spec
+from eth2spec.debug.tools import get_ssz_object_from_json_file
+
+file_dir = '<YOUR DIR PATH>'
+
+# Load JSON file from Beacon API into Remerkleable Python SSZ object
+pre_state = get_ssz_object_from_json_file(spec.BeaconState, f"{file_dir}/state_1.ssz")
+post_state = get_ssz_object_from_json_file(spec.BeaconState, f"{file_dir}/state_2.ssz")
+signed_block = get_ssz_object_from_json_file(spec.SignedBeaconBlock, f"{file_dir}/block_2.ssz")
+```
+
+### SSZ-snappy
+
+The Pyspec test generators generate test vectors to [`consensus-spec-tests`](https://github.com/ethereum/consensus-spec-tests) in [SSZ-snappy encoded format](https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/p2p-interface.md#ssz-snappy-encoding-strategy).
+
+#### Helpers
+
+##### `eth2spec.debug.tools.get_ssz_object_from_ssz_encoded`
+
+```python
+get_ssz_object_from_ssz_encoded(container: Container, file_path: str, is_snappy: bool=True) -> SSZObject
+```
+
+Get the `SSZObject` from a certain binary file.
+
+###### Arguments
+- `container: Container`: The SSZ Container class. e.g., `BeaconState`.
+- `file_path: str`: The path of the JSON file.
+- `is_snappy: bool`: If `True`, it's the SSZ-snappy encoded file; else, it's only the SSZ serialized file without snappy. Default to `True`.
+
+###### Example
+```python
+import eth2spec.capella.mainnet as spec
+from eth2spec.debug.tools import get_ssz_object_from_ssz_encoded
+
+file_dir = '<YOUR DIR PATH>'
+
+# Load SSZ-snappy file into Remerkleable Python SSZ object
+pre_state = get_ssz_object_from_ssz_encoded(spec.BeaconState, f"{file_dir}/state_1.ssz_snappy")
+post_state = get_ssz_object_from_ssz_encoded(spec.BeaconState, f"{file_dir}/state_2.ssz_snappy")
+signed_block = get_ssz_object_from_ssz_encoded(spec.SignedBeaconBlock, f"{file_dir}/block_2.ssz_snappy")
+```
+
+#### Use script to dump a specific field to a new file
+
+##### Installation
+```sh
+cd consensus-specs/scripts
+
+pip3 install -r requirements.txt
+```
+
+##### Dump SSZ to JSON file
+
+```
+python inspect_state.py --state-path=<The path of ssz_snappy state file> --output-dir=<Your output dir> --fork=capella --field=<One field of BeaconState> --is-snappy
+```
+
+- `--state-path`: The path of SSZ-snappy state file
+- `--output-dir`: The directory of the output folder
+- `--fork`: The fork name. e.g., `capella`
+- `--field`: The specific field of `BeaconState` that gets printed to the output file
+- `--is-snappy`: The flag to indicate it's SSZ-snappy. Otherwise, it's only SSZ serialized file.

scripts/inspect_state.py

@@ -0,0 +1,82 @@
+from eth2spec.test.context import spec_targets
+from eth2spec.debug.encode import encode
+from eth2spec.debug.tools import (
+    get_state_from_json_file,
+    get_state_from_ssz_encoded,
+    dump_yaml_fn,
+    output_ssz_to_file,
+)
+
+
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--state-path",
+        dest="state_path",
+        required=True,
+        help='the snappy-ed state file path',
+    )
+    parser.add_argument(
+        "-o",
+        "--output-dir",
+        dest="output_dir",
+        required=True,
+        help='the output directory',
+    )
+    parser.add_argument(
+        "--fork",
+        dest="fork",
+        default='capella',
+        help='the version of the state',
+    )
+    parser.add_argument(
+        "--preset",
+        dest="preset",
+        default='mainnet',
+        help='the version of the state',
+    )
+    parser.add_argument(
+        "--field",
+        dest="field",
+        required=False,
+        help='specific field',
+    )
+    parser.add_argument(
+        "--is-snappy",
+        dest="is_snappy",
+        help='is snappy compressed',
+        action='store_true',  # False by default
+    )
+    parser.add_argument(
+        "--is-json",
+        dest="is_json",
+        help='is JSON file from REST API',
+        action='store_true',  # False by default
+    )
+    args = parser.parse_args()
+    state_path = args.state_path
+    output_dir = args.output_dir
+    preset = args.preset
+    fork = args.fork
+    field = args.field
+    is_snappy = args.is_snappy
+    is_json = args.is_json
+
+    try:
+        spec = spec_targets[preset][fork]
+    except KeyError as e:
+        print(f'[Error] Wrong key {preset} or {fork}:')
+        print(e)
+        exit()
+
+    if is_json:
+        state = get_state_from_json_file(spec, file_path=state_path)
+    else:
+        state = get_state_from_ssz_encoded(spec, file_path=state_path, is_snappy=is_snappy)
+
+    # Output specific field to file
+    if field is not None and field in state.fields():
+        value = state.__getattr__(field)
+        output_ssz_to_file(output_dir, value, dump_yaml_fn)

scripts/requirements.txt

@@ -1 +1 @@
-../[generator]
+../[test]

setup.py

@@ -577,7 +577,7 @@ def run(self):
     cmdclass=commands,
     python_requires=">=3.9, <4",
     extras_require={
-        "test": ["pytest>=4.4", "pytest-cov", "pytest-xdist"],
+        "test": ["pytest>=4.4", "pytest-cov", "pytest-xdist", "python-snappy==0.7.3"],
         "lint": ["flake8==5.0.4", "mypy==0.981", "pylint==3.3.1", "codespell<3.0.0,>=2.0.0"],
         "generator": ["setuptools>=72.0.0", "pytest>4.4", "python-snappy==0.7.3", "filelock", "pathos==0.3.0"],
         "docs": ["mkdocs==1.4.2", "mkdocs-material==9.1.5", "mdx-truly-sane-lists==1.3",  "mkdocs-awesome-pages-plugin==2.8.0"]

tests/core/pyspec/eth2spec/debug/tools.py

@@ -0,0 +1,66 @@
+from pathlib import Path
+import json
+
+
+from ruamel.yaml import (
+    YAML,
+)
+from snappy import decompress
+
+from eth2spec.debug.encode import encode
+
+
+def load_file(file_path, mode='r'):
+    try:
+        with open(file_path, mode) as file:
+            data = file.read()
+            return data
+    except FileNotFoundError:
+        print(f"[Error] File {file_path} not found.")
+        exit()
+
+
+def get_ssz_object_from_json_file(container, file_path):
+    with open(Path(file_path), 'r') as f:
+        json_data = json.load(f)
+    return container.from_obj(json_data['data'])
+
+
+def get_state_from_json_file(spec, file_path):
+    return get_ssz_object_from_json_file(spec.BeaconState, file_path)
+
+
+def get_ssz_object_from_ssz_encoded(container, file_path, is_snappy=True):
+    state_bytes = load_file(Path(file_path), mode='rb')
+    if is_snappy:
+        state_bytes = decompress(state_bytes)
+    return container.decode_bytes(state_bytes)
+
+
+def get_state_from_ssz_encoded(spec, file_path, is_snappy=True):
+    return get_ssz_object_from_ssz_encoded(spec.BeaconState, file_path, is_snappy=is_snappy)
+
+
+def output_ssz_to_file(output_dir, value, dump_yaml_fn):
+    # output full data to file
+    yaml = YAML(pure=True)
+    yaml.default_flow_style = None
+    output_dir = Path(output_dir)
+    output_part(output_dir, dump_yaml_fn(
+        data=encode(value), name='output', file_mode="w", yaml_encoder=yaml))
+
+
+# TODO: This function will be extracted in `gen_runner.py` in https://github.com/ethereum/consensus-specs/pull/3347
+def output_part(case_dir, fn):
+    # make sure the test case directory is created before any test part is written.
+    case_dir.mkdir(parents=True, exist_ok=True)
+    fn(case_dir)
+
+
+# FIXME: duplicate to `gen_runner.py` function
+def dump_yaml_fn(data, name, file_mode, yaml_encoder):
+    def dump(case_path: Path):
+        out_path = case_path / Path(name + '.yaml')
+        with out_path.open(file_mode) as f:
+            yaml_encoder.dump(data, f)
+    return dump