Skip to content

Commit

Permalink
Merge pull request #3192 from ethereum/dev
Browse files Browse the repository at this point in the history
release for jan testnets
  • Loading branch information
djrtwo authored Jan 6, 2023
2 parents faa9fea + 3212c41 commit 5f1b88f
Show file tree
Hide file tree
Showing 34 changed files with 721 additions and 539 deletions.
133 changes: 133 additions & 0 deletions .github/workflows/run-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
name: Run spec tests and linter

defaults:
run:
shell: zsh {0}

env:
TEST_PRESET_TYPE: "minimal"
DEFAULT_BRANCH: "dev"

# Run tests on workflow_Dispatch
on:
push:
branches:
- dev
- master
pull_request:
workflow_dispatch:
inputs:
test_preset_type:
default: minimal
description: Type of test to run, either mainnet or minimal
type: string
required: true
commitRef:
description: The branch, tag or SHA to checkout and build from
default: dev
required: true
schedule:
- cron: '0 0 * * *'

jobs:
precleanup:
runs-on: self-hosted
if: always()
steps:
- name: 'Cleanup build folder'
run: |
ls -la ./
rm -rf ./* || true
rm -rf ./.??* || true
ls -la ./
setup-env:
runs-on: self-hosted
needs: precleanup
steps:
- name: Checkout this repo
uses: actions/[email protected]
with:
ref: ${{ github.event.inputs.commitRef || env.DEFAULT_BRANCH }}
- uses: actions/[email protected]
id: cache-git
with:
path: ./*
key: ${{ github.sha }}

table_of_contents:
runs-on: self-hosted
needs: setup-env
steps:
- uses: actions/[email protected]
id: restore-build
with:
path: ./*
key: ${{ github.sha }}
- name: Check table of contents
run: sudo npm install -g doctoc@2 && make check_toc

codespell:
runs-on: self-hosted
needs: setup-env
steps:
- name: Check codespell
run: pip install 'codespell<3.0.0,>=2.0.0' --user && make codespell

lint:
runs-on: self-hosted
needs: setup-env
steps:
- name: Run linter for pyspec
run: make lint
- name: Run linter for test generators
run: make lint_generators

pyspec-tests:
runs-on: self-hosted
needs: [setup-env,lint,codespell,table_of_contents]
strategy:
matrix:
version: ["phase0", "altair", "bellatrix", "capella", "eip4844"]
steps:
- uses: actions/[email protected]
id: restore-build
with:
path: ./*
key: ${{ github.sha }}
- name: set TEST_PRESET_TYPE
if: github.event.inputs.test_preset_type != ''
run: |
echo "spec_test_preset_type=${{ github.event.inputs.test_preset_type || env.TEST_PRESET_TYPE }}" >> $GITHUB_ENV
- name: set TEST_PRESET_TYPE
if: ${{ (github.event_name == 'push' && github.ref_name != 'master') || github.event_name == 'pull_request' }}
run: |
echo "spec_test_preset_type=${{ env.TEST_PRESET_TYPE}}" >> $GITHUB_ENV
- name: set TEST_PRESET_TYPE
if: ${{ github.event_name == 'push' && github.ref_name == 'master' }}
run: |
echo "spec_test_preset_type=mainnet" >> $GITHUB_ENV
- name: set TEST_PRESET_TYPE
if: github.event.schedule=='0 0 * * *'
run: |
echo "spec_test_preset_type=mainnet" >> $GITHUB_ENV
- name: Install pyspec requirements
run: make install_test
- name: test-${{ matrix.version }}
run: make citest fork=${{ matrix.version }} TEST_PRESET_TYPE=${{env.spec_test_preset_type}}
- uses: actions/upload-artifact@v3
if: always()
with:
name: test-${{ matrix.version }}
path: tests/core/pyspec/test-reports

cleanup:
runs-on: self-hosted
needs: [setup-env,pyspec-tests,codespell,lint,table_of_contents]
if: always()
steps:
- name: 'Cleanup build folder'
run: |
ls -la ./
rm -rf ./* || true
rm -rf ./.??* || true
ls -la ./
18 changes: 9 additions & 9 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ SOLIDITY_DEPOSIT_CONTRACT_SOURCE = ${SOLIDITY_DEPOSIT_CONTRACT_DIR}/deposit_cont
SOLIDITY_FILE_NAME = deposit_contract.json
DEPOSIT_CONTRACT_TESTER_DIR = ${SOLIDITY_DEPOSIT_CONTRACT_DIR}/web3_tester
CONFIGS_DIR = ./configs

TEST_PRESET_TYPE ?= minimal
# Collect a list of generator names
GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/.)))
# Map this list of generator paths to "gen_{generator name}" entries
Expand Down Expand Up @@ -96,30 +96,30 @@ generate_tests: $(GENERATOR_TARGETS)

# "make pyspec" to create the pyspec for all phases.
pyspec:
. venv/bin/activate; python3 setup.py pyspecdev
python3 -m venv venv; . venv/bin/activate; python3 setup.py pyspecdev

# installs the packages to run pyspec tests
install_test:
python3 -m venv venv; . venv/bin/activate; python3 -m pip install -e .[lint]; python3 -m pip install -e .[test]

# Testing against `minimal` config by default
# Testing against `minimal` or `mainnet` config by default
test: pyspec
. venv/bin/activate; cd $(PY_SPEC_DIR); \
python3 -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.minimal --cov=eth2spec.altair.minimal --cov=eth2spec.bellatrix.minimal --cov=eth2spec.capella.minimal --cov=eth2spec.eip4844.minimal --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
python3 -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.$(TEST_PRESET_TYPE) --cov=eth2spec.altair.$(TEST_PRESET_TYPE) --cov=eth2spec.bellatrix.$(TEST_PRESET_TYPE) --cov=eth2spec.capella.$(TEST_PRESET_TYPE) --cov=eth2spec.eip4844.$(TEST_PRESET_TYPE) --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec

# Testing against `minimal` config by default
# Testing against `minimal` or `mainnet` config by default
find_test: pyspec
. venv/bin/activate; cd $(PY_SPEC_DIR); \
python3 -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.minimal --cov=eth2spec.altair.minimal --cov=eth2spec.bellatrix.minimal --cov=eth2spec.capella.minimal --cov=eth2spec.eip4844.minimal --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
python3 -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.$(TEST_PRESET_TYPE) --cov=eth2spec.altair.$(TEST_PRESET_TYPE) --cov=eth2spec.bellatrix.$(TEST_PRESET_TYPE) --cov=eth2spec.capella.$(TEST_PRESET_TYPE) --cov=eth2spec.eip4844.$(TEST_PRESET_TYPE) --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec

citest: pyspec
mkdir -p $(TEST_REPORT_DIR);
ifdef fork
. venv/bin/activate; cd $(PY_SPEC_DIR); \
python3 -m pytest -n 4 --bls-type=milagro --fork=$(fork) --junitxml=test-reports/test_results.xml eth2spec
python3 -m pytest -n 16 --bls-type=milagro --preset=$(TEST_PRESET_TYPE) --fork=$(fork) --junitxml=test-reports/test_results.xml eth2spec
else
. venv/bin/activate; cd $(PY_SPEC_DIR); \
python3 -m pytest -n 4 --bls-type=milagro --junitxml=test-reports/test_results.xml eth2spec
python3 -m pytest -n 16 --bls-type=milagro --preset=$(TEST_PRESET_TYPE) --junitxml=test-reports/test_results.xml eth2spec
endif


Expand All @@ -135,7 +135,7 @@ check_toc: $(MARKDOWN_FILES:=.toc)
rm $*.tmp

codespell:
codespell . --skip ./.git -I .codespell-whitelist
codespell . --skip "./.git,./venv,$(PY_SPEC_DIR)/.mypy_cache" -I .codespell-whitelist

# TODO: add future protocol upgrade patch packages to linting.
# NOTE: we use `pylint` just for catching unused arguments in spec code
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ Features are researched and developed in parallel, and then consolidated into se
| Code Name or Topic | Specs | Notes |
| - | - | - |
| Capella (tentative) | <ul><li>Core</li><ul><li>[Beacon chain changes](specs/capella/beacon-chain.md)</li><li>[Capella fork](specs/capella/fork.md)</li></ul><li>Additions</li><ul><li>[Validator additions](specs/capella/validator.md)</li><li>[P2P networking](specs/capella/p2p-interface.md)</li></ul></ul> |
| EIP4844 (tentative) | <ul><li>Core</li><ul><li>[Beacon Chain changes](specs/eip4844/beacon-chain.md)</li><li>[EIP-4844 fork](specs/eip4844/fork.md)</li><li>[Polynomial commitments](specs/eip4844/polynomial-commitments.md)</li></ul><li>Additions</li><ul><li>[Honest validator guide changes](specs/eip4844/validator.md)</li><li>[P2P networking](specs/eip4844/p2p-interface.md)</li></ul></ul> |
| EIP4844 (tentative) | <ul><li>Core</li><ul><li>[Beacon Chain changes](specs/eip4844/beacon-chain.md)</li><li>[EIP-4844 fork](specs/eip4844/fork.md)</li><li>[Polynomial commitments](specs/eip4844/polynomial-commitments.md)</li><li>[Fork choice changes](specs/eip4844/fork-choice.md)</li></ul><li>Additions</li><ul><li>[Honest validator guide changes](specs/eip4844/validator.md)</li><li>[P2P networking](specs/eip4844/p2p-interface.md)</li></ul></ul> |
| Sharding (outdated) | <ul><li>Core</li><ul><li>[Beacon Chain changes](specs/sharding/beacon-chain.md)</li></ul><li>Additions</li><ul><li>[P2P networking](specs/sharding/p2p-interface.md)</li></ul></ul> |
| Custody Game (outdated) | <ul><li>Core</li><ul><li>[Beacon Chain changes](specs/custody_game/beacon-chain.md)</li></ul><li>Additions</li><ul><li>[Honest validator guide changes](specs/custody_game/validator.md)</li></ul></ul> | Dependent on sharding |
| Data Availability Sampling (outdated) | <ul><li>Core</li><ul><li>[Core types and functions](specs/das/das-core.md)</li><li>[Fork choice changes](specs/das/fork-choice.md)</li></ul><li>Additions</li><ul><li>[P2P Networking](specs/das/p2p-interface.md)</li><li>[Sampling process](specs/das/sampling.md)</li></ul></ul> | <ul><li> Dependent on sharding</li><li>[Technical explainer](https://hackmd.io/@HWeNw8hNRimMm2m2GH56Cw/B1YJPGkpD)</li></ul> |
Expand Down
29 changes: 1 addition & 28 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -638,34 +638,6 @@ def preparations(cls):
@classmethod
def sundry_functions(cls) -> str:
return super().sundry_functions() + '\n\n' + '''
#
# Temporarily disable Withdrawals functions for EIP4844 testnets
#
def no_op(fn): # type: ignore
# pylint: disable=unused-argument
def wrapper(*args, **kw): # type: ignore
return None
return wrapper
def get_empty_list_result(fn): # type: ignore
# pylint: disable=unused-argument
def wrapper(*args, **kw): # type: ignore
return []
return wrapper
process_withdrawals = no_op(process_withdrawals)
process_bls_to_execution_change = no_op(process_bls_to_execution_change)
get_expected_withdrawals = get_empty_list_result(get_expected_withdrawals)
#
# End
#
def retrieve_blobs_sidecar(slot: Slot, beacon_block_root: Root) -> PyUnion[BlobsSidecar, str]:
# pylint: disable=unused-argument
return "TEST"'''
Expand Down Expand Up @@ -1020,6 +992,7 @@ def finalize_options(self):
self.md_doc_paths += """
specs/eip4844/beacon-chain.md
specs/eip4844/fork.md
specs/eip4844/fork-choice.md
specs/eip4844/polynomial-commitments.md
specs/eip4844/p2p-interface.md
specs/eip4844/validator.md
Expand Down
66 changes: 31 additions & 35 deletions specs/altair/light-client/full-node.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
- [Introduction](#introduction)
- [Helper functions](#helper-functions)
- [`compute_merkle_proof_for_state`](#compute_merkle_proof_for_state)
- [`block_to_light_client_header`](#block_to_light_client_header)
- [Deriving light client data](#deriving-light-client-data)
- [`create_light_client_bootstrap`](#create_light_client_bootstrap)
- [`create_light_client_update`](#create_light_client_update)
Expand All @@ -34,6 +35,19 @@ def compute_merkle_proof_for_state(state: BeaconState,
...
```

### `block_to_light_client_header`

```python
def block_to_light_client_header(block: SignedBeaconBlock) -> BeaconBlockHeader:
return BeaconBlockHeader(
slot=block.message.slot,
proposer_index=block.message.proposer_index,
parent_root=block.message.parent_root,
state_root=block.message.state_root,
body_root=hash_tree_root(block.message.body),
)
```

## Deriving light client data

Full nodes are expected to derive light client data from historic blocks and states and provide it to other clients.
Expand All @@ -55,13 +69,7 @@ def create_light_client_bootstrap(state: BeaconState,
assert hash_tree_root(header) == hash_tree_root(block.message)

return LightClientBootstrap(
header=BeaconBlockHeader(
slot=state.latest_block_header.slot,
proposer_index=state.latest_block_header.proposer_index,
parent_root=state.latest_block_header.parent_root,
state_root=hash_tree_root(state),
body_root=state.latest_block_header.body_root,
),
header=block_to_light_client_header(block),
current_sync_committee=state.current_sync_committee,
current_sync_committee_branch=compute_merkle_proof_for_state(state, CURRENT_SYNC_COMMITTEE_INDEX),
)
Expand Down Expand Up @@ -103,42 +111,30 @@ def create_light_client_update(state: BeaconState,
assert hash_tree_root(attested_header) == hash_tree_root(attested_block.message) == block.message.parent_root
update_attested_period = compute_sync_committee_period_at_slot(attested_block.message.slot)

update = LightClientUpdate()

update.attested_header = block_to_light_client_header(attested_block)

# `next_sync_committee` is only useful if the message is signed by the current sync committee
if update_attested_period == update_signature_period:
next_sync_committee = attested_state.next_sync_committee
next_sync_committee_branch = compute_merkle_proof_for_state(attested_state, NEXT_SYNC_COMMITTEE_INDEX)
else:
next_sync_committee = SyncCommittee()
next_sync_committee_branch = [Bytes32() for _ in range(floorlog2(NEXT_SYNC_COMMITTEE_INDEX))]
update.next_sync_committee = attested_state.next_sync_committee
update.next_sync_committee_branch = compute_merkle_proof_for_state(
attested_state, NEXT_SYNC_COMMITTEE_INDEX)

# Indicate finality whenever possible
if finalized_block is not None:
if finalized_block.message.slot != GENESIS_SLOT:
finalized_header = BeaconBlockHeader(
slot=finalized_block.message.slot,
proposer_index=finalized_block.message.proposer_index,
parent_root=finalized_block.message.parent_root,
state_root=finalized_block.message.state_root,
body_root=hash_tree_root(finalized_block.message.body),
)
assert hash_tree_root(finalized_header) == attested_state.finalized_checkpoint.root
update.finalized_header = block_to_light_client_header(finalized_block)
assert hash_tree_root(update.finalized_header) == attested_state.finalized_checkpoint.root
else:
assert attested_state.finalized_checkpoint.root == Bytes32()
finalized_header = BeaconBlockHeader()
finality_branch = compute_merkle_proof_for_state(attested_state, FINALIZED_ROOT_INDEX)
else:
finalized_header = BeaconBlockHeader()
finality_branch = [Bytes32() for _ in range(floorlog2(FINALIZED_ROOT_INDEX))]

return LightClientUpdate(
attested_header=attested_header,
next_sync_committee=next_sync_committee,
next_sync_committee_branch=next_sync_committee_branch,
finalized_header=finalized_header,
finality_branch=finality_branch,
sync_aggregate=block.message.body.sync_aggregate,
signature_slot=block.message.slot,
)
update.finality_branch = compute_merkle_proof_for_state(
attested_state, FINALIZED_ROOT_INDEX)

update.sync_aggregate = block.message.body.sync_aggregate
update.signature_slot = block.message.slot

return update
```

Full nodes SHOULD provide the best derivable `LightClientUpdate` (according to `is_better_update`) for each sync committee period covering any epochs in range `[max(ALTAIR_FORK_EPOCH, current_epoch - MIN_EPOCHS_FOR_BLOCK_REQUESTS), current_epoch]` where `current_epoch` is defined by the current wall-clock time. Full nodes MAY also provide `LightClientUpdate` for other sync committee periods.
Expand Down
Loading

0 comments on commit 5f1b88f

Please sign in to comment.