diff --git a/docs/docs/load-ssz.md b/docs/docs/load-ssz.md new file mode 100644 index 0000000000..6ef074eb09 --- /dev/null +++ b/docs/docs/load-ssz.md @@ -0,0 +1,88 @@ +# Load SSZ file into Pyspec SSZ object + +## Common file formats + +### Beacon APIs response + +The [Beacon APIs](https://github.com/ethereum/beacon-APIs) return JSON format response. In case of a successful response, the requested object can be found within the `response['data']` field. + +#### Helpers + +##### `eth2spec.debug.tools.get_ssz_object_from_json_file` + +```python +get_ssz_object_from_json_file(container: Container, file_path: str) -> SSZObject +``` + +Get the `SSZObject` from a specific JSON file. + +###### Arguments +- `container: Container`: the SSZ Container class. e.g., `BeaconState`. +- `file_path: str`: the path of the JSON file. + +###### Example + +```python +import eth2spec.capella.mainnet as spec +from eth2spec.debug.tools import get_ssz_object_from_json_file + +file_dir = '' + +# Load JSON file from Beacon API into Remerkleable Python SSZ object +pre_state = get_ssz_object_from_json_file(spec.BeaconState, f"{file_dir}/state_1.ssz") +post_state = get_ssz_object_from_json_file(spec.BeaconState, f"{file_dir}/state_2.ssz") +signed_block = get_ssz_object_from_json_file(spec.SignedBeaconBlock, f"{file_dir}/block_2.ssz") +``` + +### SSZ-snappy + +The Pyspec test generators generate test vectors to [`consensus-spec-tests`](https://github.com/ethereum/consensus-spec-tests) in [SSZ-snappy encoded format](https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/p2p-interface.md#ssz-snappy-encoding-strategy). + +#### Helpers + +##### `eth2spec.debug.tools.get_ssz_object_from_ssz_encoded` + +```python +get_ssz_object_from_ssz_encoded(container: Container, file_path: str, is_snappy: bool=True) -> SSZObject +``` + +Get the `SSZObject` from a certain binary file. + +###### Arguments +- `container: Container`: The SSZ Container class. e.g., `BeaconState`. +- `file_path: str`: The path of the JSON file. +- `is_snappy: bool`: If `True`, it's the SSZ-snappy encoded file; else, it's only the SSZ serialized file without snappy. Default to `True`. + +###### Example +```python +import eth2spec.capella.mainnet as spec +from eth2spec.debug.tools import get_ssz_object_from_ssz_encoded + +file_dir = '' + +# Load SSZ-snappy file into Remerkleable Python SSZ object +pre_state = get_ssz_object_from_ssz_encoded(spec.BeaconState, f"{file_dir}/state_1.ssz_snappy") +post_state = get_ssz_object_from_ssz_encoded(spec.BeaconState, f"{file_dir}/state_2.ssz_snappy") +signed_block = get_ssz_object_from_ssz_encoded(spec.SignedBeaconBlock, f"{file_dir}/block_2.ssz_snappy") +``` + +#### Use script to dump a specific field to a new file + +##### Installation +```sh +cd consensus-specs/scripts + +pip3 install -r requirements.txt +``` + +##### Dump SSZ to JSON file + +``` +python inspect_state.py --state-path= --output-dir= --fork=capella --field= --is-snappy +``` + +- `--state-path`: The path of SSZ-snappy state file +- `--output-dir`: The directory of the output folder +- `--fork`: The fork name. e.g., `capella` +- `--field`: The specific field of `BeaconState` that gets printed to the output file +- `--is-snappy`: The flag to indicate it's SSZ-snappy. Otherwise, it's only SSZ serialized file. diff --git a/scripts/inspect_state.py b/scripts/inspect_state.py new file mode 100644 index 0000000000..2cf4394e0e --- /dev/null +++ b/scripts/inspect_state.py @@ -0,0 +1,82 @@ +from eth2spec.test.context import spec_targets +from eth2spec.debug.encode import encode +from eth2spec.debug.tools import ( + get_state_from_json_file, + get_state_from_ssz_encoded, + dump_yaml_fn, + output_ssz_to_file, +) + + +if __name__ == '__main__': + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "--state-path", + dest="state_path", + required=True, + help='the snappy-ed state file path', + ) + parser.add_argument( + "-o", + "--output-dir", + dest="output_dir", + required=True, + help='the output directory', + ) + parser.add_argument( + "--fork", + dest="fork", + default='capella', + help='the version of the state', + ) + parser.add_argument( + "--preset", + dest="preset", + default='mainnet', + help='the version of the state', + ) + parser.add_argument( + "--field", + dest="field", + required=False, + help='specific field', + ) + parser.add_argument( + "--is-snappy", + dest="is_snappy", + help='is snappy compressed', + action='store_true', # False by default + ) + parser.add_argument( + "--is-json", + dest="is_json", + help='is JSON file from REST API', + action='store_true', # False by default + ) + args = parser.parse_args() + state_path = args.state_path + output_dir = args.output_dir + preset = args.preset + fork = args.fork + field = args.field + is_snappy = args.is_snappy + is_json = args.is_json + + try: + spec = spec_targets[preset][fork] + except KeyError as e: + print(f'[Error] Wrong key {preset} or {fork}:') + print(e) + exit() + + if is_json: + state = get_state_from_json_file(spec, file_path=state_path) + else: + state = get_state_from_ssz_encoded(spec, file_path=state_path, is_snappy=is_snappy) + + # Output specific field to file + if field is not None and field in state.fields(): + value = state.__getattr__(field) + output_ssz_to_file(output_dir, value, dump_yaml_fn) diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 98eca5969d..5706a9a091 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -1 +1 @@ -../[generator] +../[test] diff --git a/setup.py b/setup.py index 55f1d0e344..03c467f357 100644 --- a/setup.py +++ b/setup.py @@ -577,7 +577,7 @@ def run(self): cmdclass=commands, python_requires=">=3.9, <4", extras_require={ - "test": ["pytest>=4.4", "pytest-cov", "pytest-xdist"], + "test": ["pytest>=4.4", "pytest-cov", "pytest-xdist", "python-snappy==0.7.3"], "lint": ["flake8==5.0.4", "mypy==0.981", "pylint==3.3.1", "codespell<3.0.0,>=2.0.0"], "generator": ["setuptools>=72.0.0", "pytest>4.4", "python-snappy==0.7.3", "filelock", "pathos==0.3.0"], "docs": ["mkdocs==1.4.2", "mkdocs-material==9.1.5", "mdx-truly-sane-lists==1.3", "mkdocs-awesome-pages-plugin==2.8.0"] diff --git a/tests/core/pyspec/eth2spec/debug/tools.py b/tests/core/pyspec/eth2spec/debug/tools.py new file mode 100644 index 0000000000..d052b11cad --- /dev/null +++ b/tests/core/pyspec/eth2spec/debug/tools.py @@ -0,0 +1,66 @@ +from pathlib import Path +import json + + +from ruamel.yaml import ( + YAML, +) +from snappy import decompress + +from eth2spec.debug.encode import encode + + +def load_file(file_path, mode='r'): + try: + with open(file_path, mode) as file: + data = file.read() + return data + except FileNotFoundError: + print(f"[Error] File {file_path} not found.") + exit() + + +def get_ssz_object_from_json_file(container, file_path): + with open(Path(file_path), 'r') as f: + json_data = json.load(f) + return container.from_obj(json_data['data']) + + +def get_state_from_json_file(spec, file_path): + return get_ssz_object_from_json_file(spec.BeaconState, file_path) + + +def get_ssz_object_from_ssz_encoded(container, file_path, is_snappy=True): + state_bytes = load_file(Path(file_path), mode='rb') + if is_snappy: + state_bytes = decompress(state_bytes) + return container.decode_bytes(state_bytes) + + +def get_state_from_ssz_encoded(spec, file_path, is_snappy=True): + return get_ssz_object_from_ssz_encoded(spec.BeaconState, file_path, is_snappy=is_snappy) + + +def output_ssz_to_file(output_dir, value, dump_yaml_fn): + # output full data to file + yaml = YAML(pure=True) + yaml.default_flow_style = None + output_dir = Path(output_dir) + output_part(output_dir, dump_yaml_fn( + data=encode(value), name='output', file_mode="w", yaml_encoder=yaml)) + + +# TODO: This function will be extracted in `gen_runner.py` in https://github.com/ethereum/consensus-specs/pull/3347 +def output_part(case_dir, fn): + # make sure the test case directory is created before any test part is written. + case_dir.mkdir(parents=True, exist_ok=True) + fn(case_dir) + + +# FIXME: duplicate to `gen_runner.py` function +def dump_yaml_fn(data, name, file_mode, yaml_encoder): + def dump(case_path: Path): + out_path = case_path / Path(name + '.yaml') + with out_path.open(file_mode) as f: + yaml_encoder.dump(data, f) + return dump