diff --git a/interchain-security b/interchain-security index 1ce43a7..aee9cc6 160000 --- a/interchain-security +++ b/interchain-security @@ -1 +1 @@ -Subproject commit 1ce43a79a81bef24b112e74d87ba2e0960b8d445 +Subproject commit aee9cc6600c79941c0e4ddd93e0065b2bea68f39 diff --git a/start-docker.sh b/start-docker.sh index 578d5a2..8213aa2 100644 --- a/start-docker.sh +++ b/start-docker.sh @@ -1,18 +1,20 @@ #!/bin/bash -# set -eux +# If -e is not set then if the build fails, it will use the old container, resulting in a very confusing debugging situation +# Setting -e makes it error out if the build fails +set -eux CONTAINER_NAME=$1 INSTANCE_NAME=$2 # Must be in this format "-p 9090:9090 -p 26657:26657 -p 1317:1317 -p 8545:8545" EXPOSE_PORTS=$3 -# Build the Docker container -docker build -t "$CONTAINER_NAME" . - # Remove existing container instance set +e docker rm -f "$INSTANCE_NAME" set -e +# Build the Docker container +docker build -t "$CONTAINER_NAME" . + # Run new test container instance docker run --name "$INSTANCE_NAME" --cap-add=NET_ADMIN $EXPOSE_PORTS "$CONTAINER_NAME" /bin/bash /testnet-scripts/beacon.sh \ No newline at end of file diff --git a/testnet-scripts/start-chain/setup-validators.sh b/testnet-scripts/start-chain/setup-validators.sh deleted file mode 100644 index 97ec919..0000000 --- a/testnet-scripts/start-chain/setup-validators.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash -set -eux - -BIN=$1 -MNEMONICS=$2 -CHAIN_ID=$3 -CHAIN_IP_PREFIX=$4 -GENESIS_TRANSFORM=$5 -ALLOCATION=$6 -STAKE_AMOUNT=$7 -SKIP_GENTX=$8 - -# Get number of nodes from length of mnemonics array -NODES=$(jq '. | length' <<< "$MNEMONICS") - - -# first we start a genesis.json with validator0 -# validator0 will also collect the gentx's once gnerated -# todo add git hash to chain name -jq -r ".[0]" <<< "$MNEMONICS" | $BIN init --home /$CHAIN_ID/validator0 --chain-id=$CHAIN_ID validator0 --recover > /dev/null - - -## Modify generated genesis.json to our liking by editing fields using jq -## we could keep a hardcoded genesis file around but that would prevent us from -## testing the generated one with the default values provided by the module. - -# Apply transformations to genesis file -jq "$GENESIS_TRANSFORM" /$CHAIN_ID/validator0/config/genesis.json > /$CHAIN_ID/edited-genesis.json - -mv /$CHAIN_ID/edited-genesis.json /$CHAIN_ID/genesis.json - - -# Sets up an arbitrary number of validators on a single machine by manipulating -# the --home parameter on gaiad -for i in $(seq 0 $(($NODES - 1))); -do - # TODO: we need to pass in an identifier to identify the validator folder and other things instead of - # using the index - - # make the folders for this validator - mkdir -p /$CHAIN_ID/validator$i/config/ - - ARGS="--home /$CHAIN_ID/validator$i --keyring-backend test" - - # Generate a validator key, orchestrator key, and eth key for each validator - jq -r ".[$((i-1))]" <<< "$MNEMONICS" | $BIN keys add $ARGS validator$i --recover > /$CHAIN_ID/validator$i/mnemonic - - echo "validator$i keys:" - $BIN keys show validator$i $ARGS - - # move the genesis in - mv /$CHAIN_ID/genesis.json /$CHAIN_ID/validator$i/config/genesis.json - $BIN add-genesis-account $ARGS validator$i $ALLOCATION - - # move the genesis back out - mv /$CHAIN_ID/validator$i/config/genesis.json /$CHAIN_ID/genesis.json -done - - -for i in $(seq 0 $(($NODES - 1))); -do - cp /$CHAIN_ID/genesis.json /$CHAIN_ID/validator$i/config/genesis.json - - $BIN gentx validator$i "$STAKE_AMOUNT" --home /$CHAIN_ID/validator$i --keyring-backend test --moniker validator$i --chain-id=$CHAIN_ID --ip $CHAIN_IP_PREFIX.$i - - # obviously we don't need to copy validator0's gentx to itself - if [ $i -gt 0 ]; then - cp /$CHAIN_ID/validator$i/config/gentx/* /$CHAIN_ID/validator0/config/gentx/ - fi -done - -if [ "$SKIP_GENTX" = "false" ] ; then - # make the final genesis.json - $BIN collect-gentxs --home /$CHAIN_ID/validator0 -fi - -# and copy it to the root -cp /$CHAIN_ID/validator0/config/genesis.json /$CHAIN_ID/genesis.json - -# put the now final genesis.json into the correct folders -for i in $(seq 1 $(($NODES - 1))); -do - cp /$CHAIN_ID/genesis.json /$CHAIN_ID/validator$i/config/genesis.json -done - diff --git a/testnet-scripts/start-chain/start-chain.sh b/testnet-scripts/start-chain/start-chain.sh index b605fa0..e37b0f2 100644 --- a/testnet-scripts/start-chain/start-chain.sh +++ b/testnet-scripts/start-chain/start-chain.sh @@ -1,14 +1,17 @@ #!/bin/bash set -eux -# the directory of this script -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - # The gaiad binary BIN=$1 -# Mnemonics with which to start nodes -MNEMONICS=$2 +# JSON array of validator information +# [{ +# mnemonic: "crackle snap pop ... etc", +# allocation: "10000000000stake,10000000000footoken", +# stake: "5000000000stake", +# number: "0" +# }, ... ] +VALIDATORS=$2 # The chain ID CHAIN_ID=$3 @@ -18,32 +21,161 @@ CHAIN_ID=$3 # For example: "7.7.7" CHAIN_IP_PREFIX=$4 -# Default: 26657 -RPC_PORT=$5 +# A transformation to apply to the genesis file, as a jq string +GENESIS_TRANSFORM=$5 -# Default: 9090 -GRPC_PORT=$6 +# Whether to skip collecting gentxs so that the genesis does not have them +SKIP_GENTX=$6 -# A transformation to apply to the genesis file, as a jq string -GENESIS_TRANSFORM=$7 +# Whether to copy in validator configs from somewhere else +COPY_KEYS=$7 -# How much coin to give each validator on start -# Default: "10000000000stake,10000000000footoken" -ALLOCATION=$8 -# Amount for each validator to stake -STAKE_AMOUNT=$9 -# Whether to skip collecting gentxs so that the genesis does not have them -SKIP_GENTX=${10} +# CREATE VALIDATORS AND DO GENESIS CEREMONY + +# Get number of nodes from length of validators array +NODES=$(echo "$VALIDATORS" | jq '. | length') + +# first we start a genesis.json with the first validator +# the first validator will also collect the gentx's once gnerated +FIRST_VAL_ID=$(echo "$VALIDATORS" | jq -r ".[0].number") +echo "$VALIDATORS" | jq -r ".[0].mnemonic" | $BIN init --home /$CHAIN_ID/validator$FIRST_VAL_ID --chain-id=$CHAIN_ID validator$FIRST_VAL_ID --recover > /dev/null + +# Apply jq transformations to genesis file +jq "$GENESIS_TRANSFORM" /$CHAIN_ID/validator$FIRST_VAL_ID/config/genesis.json > /$CHAIN_ID/edited-genesis.json +mv /$CHAIN_ID/edited-genesis.json /$CHAIN_ID/genesis.json + + + + +# CREATE VALIDATOR HOME FOLDERS ETC + +for i in $(seq 0 $(($NODES - 1))); +do + VAL_ID=$(echo "$VALIDATORS" | jq -r ".[$i].number") + # make the folders for this validator + mkdir -p /$CHAIN_ID/validator$VAL_ID/config/ + + # Generate an application key for each validator + # Sets up an arbitrary number of validators on a single machine by manipulating + # the --home parameter on gaiad + echo "$VALIDATORS" | jq -r ".[$i].mnemonic" | $BIN keys add validator$VAL_ID \ + --home /$CHAIN_ID/validator$VAL_ID \ + --keyring-backend test \ + --recover > /dev/null + + # Give validators their initial token allocations + # move the genesis in + mv /$CHAIN_ID/genesis.json /$CHAIN_ID/validator$VAL_ID/config/genesis.json + + # give this validator some money + ALLOCATION=$(echo "$VALIDATORS" | jq -r ".[$i].allocation") + $BIN add-genesis-account validator$VAL_ID $ALLOCATION \ + --home /$CHAIN_ID/validator$VAL_ID \ + --keyring-backend test + + # move the genesis back out + mv /$CHAIN_ID/validator$VAL_ID/config/genesis.json /$CHAIN_ID/genesis.json +done + + + +# MAKE GENTXS AND SET UP LOCAL VALIDATOR STATE + +for i in $(seq 0 $(($NODES - 1))); +do + VAL_ID=$(echo "$VALIDATORS" | jq -r ".[$i].number") + # Copy in the genesis.json + cp /$CHAIN_ID/genesis.json /$CHAIN_ID/validator$VAL_ID/config/genesis.json + + # Make a gentx (this command also sets up validator state on disk even if we are not going to use the gentx for anything) + STAKE_AMOUNT=$(echo "$VALIDATORS" | jq -r ".[$i].stake") + $BIN gentx validator$VAL_ID "$STAKE_AMOUNT" \ + --home /$CHAIN_ID/validator$VAL_ID \ + --keyring-backend test \ + --moniker validator$VAL_ID \ + --chain-id=$CHAIN_ID \ + --ip $CHAIN_IP_PREFIX.$VAL_ID + + # Copy gentxs to the first validator for possible future collection. + # Obviously we don't need to copy the first validator's gentx to itself + if [ $VAL_ID != $FIRST_VAL_ID ]; then + cp /$CHAIN_ID/validator$VAL_ID/config/gentx/* /$CHAIN_ID/validator$FIRST_VAL_ID/config/gentx/ + fi + + # Copy in keys from another chain. This is used to start a consumer chain + if [ "$COPY_KEYS" != "" ] ; then + cp /$COPY_KEYS/validator$VAL_ID/config/priv_validator_key.json /$CHAIN_ID/validator$VAL_ID/config/ + cp /$COPY_KEYS/validator$VAL_ID/config/node_key.json /$CHAIN_ID/validator$VAL_ID/config/ + fi +done + + + + +# COLLECT GENTXS IF WE ARE STARTING A NEW CHAIN + +if [ "$SKIP_GENTX" = "false" ] ; then + # make the final genesis.json + $BIN collect-gentxs --home /$CHAIN_ID/validator$FIRST_VAL_ID + + # and copy it to the root + cp /$CHAIN_ID/validator$FIRST_VAL_ID/config/genesis.json /$CHAIN_ID/genesis.json + + # put the now final genesis.json into the correct folders + for i in $(seq 1 $(($NODES - 1))); + do + VAL_ID=$(echo "$VALIDATORS" | jq -r ".[$i].number") + cp /$CHAIN_ID/genesis.json /$CHAIN_ID/validator$VAL_ID/config/genesis.json + done +fi + + + + +# START VALIDATOR NODES + +for i in $(seq 0 $(($NODES - 1))); +do + VAL_ID=$(echo "$VALIDATORS" | jq -r ".[$i].number") + # add this ip for loopback dialing + ip addr add $CHAIN_IP_PREFIX.$VAL_ID/32 dev eth0 || true # allowed to fail + + GAIA_HOME="--home /$CHAIN_ID/validator$VAL_ID" + RPC_ADDRESS="--rpc.laddr tcp://$CHAIN_IP_PREFIX.$VAL_ID:26658" + GRPC_ADDRESS="--grpc.address $CHAIN_IP_PREFIX.$VAL_ID:9091" + LISTEN_ADDRESS="--address tcp://$CHAIN_IP_PREFIX.$VAL_ID:26655" + P2P_ADDRESS="--p2p.laddr tcp://$CHAIN_IP_PREFIX.$VAL_ID:26656" + LOG_LEVEL="--log_level info" + ENABLE_WEBGRPC="--grpc-web.enable=false" + + PERSISTENT_PEERS="" + + for j in $(seq 0 $(($NODES - 1))); + do + if [ $i -ne $j ]; then + PEER_VAL_ID=$(echo "$VALIDATORS" | jq -r ".[$j].number") + NODE_ID=$($BIN tendermint show-node-id --home /$CHAIN_ID/validator$PEER_VAL_ID) + ADDRESS="$NODE_ID@$CHAIN_IP_PREFIX.$PEER_VAL_ID:26656" + # (jq -r '.body.memo' /$CHAIN_ID/validator$j/config/gentx/*) # Getting the address from the gentx should also work + PERSISTENT_PEERS="$PERSISTENT_PEERS,$ADDRESS" + fi + done + + # Remove leading comma and concat to flag + PERSISTENT_PEERS="--p2p.persistent_peers ${PERSISTENT_PEERS:1}" + + ARGS="$GAIA_HOME $LISTEN_ADDRESS $RPC_ADDRESS $GRPC_ADDRESS $LOG_LEVEL $P2P_ADDRESS $ENABLE_WEBGRPC $PERSISTENT_PEERS" + $BIN $ARGS start &> /$CHAIN_ID/validator$VAL_ID/logs & +done + + -# generate accounts and do genesis ceremony -/bin/bash "$DIR/setup-validators.sh" "$BIN" "$MNEMONICS" "$CHAIN_ID" "$CHAIN_IP_PREFIX" "$GENESIS_TRANSFORM" "$ALLOCATION" "$STAKE_AMOUNT" "$SKIP_GENTX" -/bin/bash "$DIR/start-validators.sh" "$BIN" "$MNEMONICS" "$CHAIN_ID" "$CHAIN_IP_PREFIX" "$RPC_PORT" "$GRPC_PORT" # poll for chain start set +e -until interchain-securityd query block --node "tcp://$CHAIN_IP_PREFIX.0:26658" | grep -q -v '{"block_id":{"hash":"","parts":{"total":0,"hash":""}},"block":null}'; do sleep 0.3 ; done +until interchain-securityd query block --node "tcp://$CHAIN_IP_PREFIX.$FIRST_VAL_ID:26658" | grep -q -v '{"block_id":{"hash":"","parts":{"total":0,"hash":""}},"block":null}'; do sleep 0.3 ; done set -e echo "done!!!!!!!!" diff --git a/testnet-scripts/start-chain/start-validators.sh b/testnet-scripts/start-chain/start-validators.sh deleted file mode 100644 index 031ff61..0000000 --- a/testnet-scripts/start-chain/start-validators.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -set -eux - -BIN=$1 -MNEMONICS=$2 -CHAIN_ID=$3 - -# This is the first 3 fields of the IP addresses which will be used internally by the validators of this blockchain -# Recommended to use something starting with 7, since it is squatted by the DoD and is unroutable on the internet -# For example: "7.7.7" -CHAIN_IP_PREFIX=$4 - -# Default: 26657 -RPC_PORT=$5 -# Default: 9090 -GRPC_PORT=$6 - -# Get number of nodes from length of mnemonics array -NODES=$(jq '. | length' <<< "$MNEMONICS") - -for i in $(seq 0 $(($NODES - 1))); -do - # add this ip for loopback dialing - ip addr add $CHAIN_IP_PREFIX.$i/32 dev eth0 || true # allowed to fail - - GAIA_HOME="--home /$CHAIN_ID/validator$i" - - RPC_ADDRESS="--rpc.laddr tcp://$CHAIN_IP_PREFIX.$i:26658" - GRPC_ADDRESS="--grpc.address $CHAIN_IP_PREFIX.$i:9091" - LISTEN_ADDRESS="--address tcp://$CHAIN_IP_PREFIX.$i:26655" - P2P_ADDRESS="--p2p.laddr tcp://$CHAIN_IP_PREFIX.$i:26656" - LOG_LEVEL="--log_level info" - ENABLE_WEBGRPC="--grpc-web.enable=false" - - # only add validator1 as persistent peer, and don't add it to itself since nodes seem to error when given themselves as persistent peers - PERSISTENT_PEERS="" - if [ $i -gt 0 ]; then - PERSISTENT_PEERS="--p2p.persistent_peers $(paste -sd ',' <<< $(jq -r '.body.memo' /$CHAIN_ID/validator0/config/gentx/*))" - fi - - # SEED=$(paste -sd ',' <<< $(jq -r '.body.memo' /$CHAIN_ID/seed/config/gentx/*)) - - ARGS="$GAIA_HOME $LISTEN_ADDRESS $RPC_ADDRESS $GRPC_ADDRESS $LOG_LEVEL $P2P_ADDRESS $ENABLE_WEBGRPC $PERSISTENT_PEERS" - $BIN $ARGS start &> /$CHAIN_ID/validator$i/logs & -done diff --git a/tests/actions.go b/tests/actions.go index ce2e023..28ab410 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -17,6 +17,7 @@ type StartChainAction struct { validators []uint genesisChanges string skipGentx bool + copyConfigs string } type StartConsumerChainAction struct { @@ -70,6 +71,7 @@ type CreateChildChainProposalJSON struct { } func (s System) sendTokens(action SendTokensAction) { + println("FROM ADDRESS", action.from, s.validatorConfigs[action.from].delAddress) bz, err := exec.Command("docker", "exec", s.containerConfig.instanceName, s.containerConfig.binaryName, "tx", "bank", "send", @@ -94,13 +96,24 @@ func (s System) startChain( action StartChainAction, ) { chainConfig := s.chainConfigs[action.chain] - var mnemonics []string + type jsonValAttrs struct { + Mnemonic string `json:"mnemonic"` + Allocation string `json:"allocation"` + Stake string `json:"stake"` + Number string `json:"number"` + } + var validators []jsonValAttrs for _, val := range action.validators { - mnemonics = append(mnemonics, s.validatorConfigs[val].mnemonic) + validators = append(validators, jsonValAttrs{ + Mnemonic: s.validatorConfigs[val].mnemonic, + Allocation: chainConfig.initialAllocation, + Stake: chainConfig.stakeAmount, + Number: fmt.Sprint(val), + }) } - mnz, err := json.Marshal(mnemonics) + vals, err := json.Marshal(validators) if err != nil { log.Fatal(err) } @@ -113,9 +126,9 @@ func (s System) startChain( } cmd := exec.Command("docker", "exec", s.containerConfig.instanceName, "/bin/bash", - "/testnet-scripts/start-chain/start-chain.sh", s.containerConfig.binaryName, string(mnz), chainConfig.chainId, chainConfig.ipPrefix, - fmt.Sprint(chainConfig.rpcPort), fmt.Sprint(chainConfig.grpcPort), genesisChanges, - chainConfig.initialAllocation, chainConfig.stakeAmount, fmt.Sprint(action.skipGentx)) + "/testnet-scripts/start-chain/start-chain.sh", s.containerConfig.binaryName, string(vals), + chainConfig.chainId, chainConfig.ipPrefix, genesisChanges, + fmt.Sprint(action.skipGentx), action.copyConfigs) cmdReader, err := cmd.StdoutPipe() if err != nil { @@ -264,6 +277,7 @@ func (s System) startConsumerChain(action StartConsumerChainAction) { validators: action.validators, genesisChanges: ".app_state.ccvchild = " + string(bz), skipGentx: true, + copyConfigs: s.chainConfigs[action.providerChain].chainId, }) } diff --git a/tests/config.go b/tests/config.go index 0d66648..640a4d0 100644 --- a/tests/config.go +++ b/tests/config.go @@ -54,15 +54,15 @@ func DefaultSystemConfig() System { validatorConfigs: []ValidatorConfig{ { mnemonic: "pave immune ethics wrap gain ceiling always holiday employ earth tumble real ice engage false unable carbon equal fresh sick tattoo nature pupil nuclear", - delAddress: "cosmos19hz4m226ztankqramvt4a7t0shejv4dc79gp9u", + delAddress: "cosmos19pe9pg5dv9k5fzgzmsrgnw9rl9asf7ddwhu7lm", }, { mnemonic: "glass trip produce surprise diamond spin excess gaze wash drum human solve dress minor artefact canoe hard ivory orange dinner hybrid moral potato jewel", - delAddress: "cosmos19pe9pg5dv9k5fzgzmsrgnw9rl9asf7ddwhu7lm", + delAddress: "cosmos1dkas8mu4kyhl5jrh4nzvm65qz588hy9qcz08la", }, { mnemonic: "sight similar better jar bitter laptop solve fashion father jelly scissors chest uniform play unhappy convince silly clump another conduct behave reunion marble animal", - delAddress: "cosmos1dkas8mu4kyhl5jrh4nzvm65qz588hy9qcz08la", + delAddress: "cosmos19hz4m226ztankqramvt4a7t0shejv4dc79gp9u", }, }, chainConfigs: []ChainConfig{ diff --git a/tests/steps.go b/tests/steps.go index 8267157..3071775 100644 --- a/tests/steps.go +++ b/tests/steps.go @@ -75,7 +75,7 @@ var exampleSteps1 = []Step{ { action: StartChainAction{ chain: 0, - validators: []uint{0, 1, 2}, + validators: []uint{1, 0, 2}, }, state: State{ 0: ChainState{ @@ -158,7 +158,7 @@ var exampleSteps1 = []Step{ action: StartConsumerChainAction{ consumerChain: 1, providerChain: 0, - validators: []uint{0, 1, 2}, + validators: []uint{2, 1, 0}, }, state: State{ 0: ChainState{