Skip to content

Commit

Permalink
Documentation/helper script changes done in 'origin/develop' deployed…
Browse files Browse the repository at this point in the history
… to _master
  • Loading branch information
vogel76 committed Oct 12, 2022
2 parents 70d79ef + 0325373 commit 80f5c4f
Show file tree
Hide file tree
Showing 82 changed files with 1,069 additions and 106 deletions.
22 changes: 14 additions & 8 deletions .gitlab-ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,9 @@ cli_wallet_tests:
script:
- cd tests/functional/python_tests/cli_wallet
- *run-pytest
# RUN TESTS ON HF25. REMOVE AFTER HARDFORK26!!!
- export HIVE_HF26_TIME=1906530101 # Distant future (1 June 2030), to make sure that node is in HF25.
- *run-pytest
tags:
- public-runner-docker
artifacts:
Expand Down Expand Up @@ -322,10 +325,13 @@ cli_wallet_extended_tests:
artifacts: true
image: "$CI_REGISTRY_IMAGE/test$TEST_IMAGE_TAG"
variables:
PYTEST_TIMEOUT_MINUTES: 11
PYTEST_TIMEOUT_MINUTES: 14
script:
- cd tests/functional/python_tests/cli_wallet_extended_tests
- *run-pytest
# RUN TESTS ON HF25. REMOVE AFTER HARDFORK26!!!
- export HIVE_HF26_TIME=1906530101 # Distant future (1 June 2030), to make sure that node is in HF25.
- *run-pytest
artifacts:
reports:
junit: tests/functional/python_tests/cli_wallet_extended_tests/report.xml
Expand All @@ -352,7 +358,7 @@ fork_tests:

hf26_tests:
stage: test
extends: .test_tools_based
extends: .test_tools_based_with_faketime
needs:
- job: testnet_node_build
artifacts: true
Expand Down Expand Up @@ -388,7 +394,7 @@ live_sync_tests:
tags:
- public-runner-docker

network_broadcast_tests:
broadcast_tests:
stage: test
extends: .test_tools_based
needs:
Expand All @@ -398,11 +404,11 @@ network_broadcast_tests:
variables:
PYTEST_TIMEOUT_MINUTES: 6
script:
- cd tests/functional/python_tests/network_broadcast_tests
- cd tests/functional/python_tests/broadcast_tests
- *run-pytest
artifacts:
reports:
junit: tests/functional/python_tests/network_broadcast_tests/report.xml
junit: tests/functional/python_tests/broadcast_tests/report.xml
tags:
- public-runner-docker

Expand Down Expand Up @@ -545,7 +551,7 @@ transaction_serialization_testnet_tests:
tags:
- public-runner-docker

transaction_serialization_mainnet_tests:
python_pattern_mainnet_tests:
stage: test
image: "$CI_REGISTRY_IMAGE/python$PYTHON_BULLSEYE_TAG"
interruptible: true
Expand All @@ -568,14 +574,14 @@ transaction_serialization_mainnet_tests:

script:
# run pattern tests
- cd $CI_PROJECT_DIR/tests/functional/python_tests/transaction_serialization_tests/cli_wallet
- cd $CI_PROJECT_DIR/tests/api_tests/python_patterns_tests
- export PYTEST_ARGS=(-m mainnet_5m --http-endpoint hived-instance:8090 --ws-endpoint hived-instance:8090 --wallet-path $CI_PROJECT_DIR/hived-mainnet-binaries/cli_wallet)
- *run-pytest

artifacts:
name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
reports:
junit: $CI_PROJECT_DIR/tests/functional/python_tests/transaction_serialization_tests/cli_wallet/report.xml
junit: $CI_PROJECT_DIR/tests/api_tests/python_patterns_tests/report.xml
paths:
- "**/generated_during_*"
- "**/generated_by_package_fixtures"
Expand Down
5 changes: 5 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,9 @@ ENV WS_PORT=${WS_PORT}
ARG HTTP_PORT=8090
ENV HTTP_PORT=${HTTP_PORT}

ARG CLI_WALLET_PORT=8093
ENV CLI_WALLET_PORT=${CLI_WALLET_PORT}

SHELL ["/bin/bash", "-c"]

USER hived
Expand Down Expand Up @@ -123,6 +126,8 @@ EXPOSE ${P2P_PORT}
EXPOSE ${WS_PORT}
# JSON rpc service
EXPOSE ${HTTP_PORT}
# Port specific to HTTP cli_wallet server
EXPOSE ${CLI_WALLET_PORT}

FROM ${CI_REGISTRY_IMAGE}instance-5m:instance-${BUILD_IMAGE_TAG} as data

Expand Down
45 changes: 44 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,12 @@ But if you would still like to build from source, we also have [build instructio

Building a docker image is described here: [Building under Docker](https://gitlab.syncad.com/hive/hive/-/blob/master/doc/building.md#building-under-docker)

If you'd like to use our already pre-built official binary images, it's as simple as downloading it from the Dockerhub registry with only one command:

```
docker pull hiveio/hive
```

To run a Hive consensus node there are needed resources:
- data directory to hold a blockchain file(s) (ca 400GB is required)
- storage to hold a shared memory file (ca. 24GB of memory is required at the moment to store state data):
Expand Down Expand Up @@ -123,9 +129,46 @@ General usage: `run_hived_img.sh <docker_img> [OPTION[=VALUE]]... [<hived_option

Other options can be as usually passed directly to the (run_hived_img.sh) command line (as hived-options) or explicitly specified in config.ini file.

4. Running API node and associated cli_wallet in network daemon mode
To run an API node and next start a cli_wallet cooperating to it, use following steps:
- start a Hive container instance running hived enabled required set of plugins, like also having additionally mapped the port cli_wallet will be operating on. It is important to specify this optional port mapping at container start, since it can't be changed later, without restarting the container.
```
./run_hived_img.sh my-local-image --name=api-instance --data-dir=/home/hived/datadir --shared-file-dir=/dev/shm/hived/api_node \
--webserver-http-endpoint=0.0.0.0:8091 --webserver-ws-endpoint=0.0.0.0:8090 --p2p-endpoint=0.0.0.0:2001 \
--docker-option="-p 8093:8093" \
--plugin=account_history_api --plugin=database_api --plugin=condenser_api --plugin=account_by_key_api --plugin=wallet_bridge_api \
--force-replay
```
- run a cli_wallet tool, which should execute **inside** docker container started in previous step
```
./run_cli_wallet_img.sh api-instance \
--rpc-http-endpoint=0.0.0.0:8093 # that means that we want to expose from docker container the cli_wallet HTTP server at port 8093. This port value should match to the port specified at `--docker-option="-p 8093:8093"` passed to run_hived_img.sh
--rpc-http-allowip=172.17.0.1 # this is very important option, which allows to accept HTTP connections (incoming to cli_wallet) only from given IPs. This example value covers default docker bridge network address mode. Best to check given ip using `docker container inspect api-instance` and verify IP specific to its Networks property
```
By default, the run_cli_wallet_img.sh script will instruct a cli_wallet to load/store its data (like password, imported keys to operate on) in the `wallet.json` file located in the directory specified as data-dir mapping during run_hived_img.sh spawn. You can also copy an existing wallet.json file to the data-dir. Then started cli_wallet will load such preconfigured values from it.
Then you can operate with cli_wallet using network API calls, like this:
curl --request POST --url http://<your-container-ip>:8093/ --header 'Content-Type: application/json' --data '{"jsonrpc": "2.0", "id": 0, "method": "info", "params": []}'
curl --request POST --url http://<your-container-ip>:8093/ --header 'Content-Type: application/json' --data '{"jsonrpc": "2.0", "id": 0, "method": "unlock", "params": ["my-secres-pass"]}'
curl --request POST --url http://<your-container-ip>:8093/ --header 'Content-Type: application/json' --data '{"jsonrpc": "2.0", "id": 0, "method": "create_account", "params": ["initminer", "alice", "{}", True]}'
## CLI Wallet
We provide a basic cli wallet for interfacing with `hived`. The wallet is self-documented via command line help. The node you connect to via the cli wallet needs to be running the `account_by_key_api`, `condenser_api`, `wallet_bridge_api` and needs to be configured to accept WebSocket connections via `webserver-ws-endpoint`.
We provide a basic cli wallet for interfacing with `hived`. The wallet is self-documented via command line help. The node you connect to via the cli wallet needs to be running the `account_by_key_api`, `condenser_api`, `database_api`, `account_history_api`, `wallet_bridge_api` and needs to be configured to accept WebSocket connections via `webserver-ws-endpoint`.
cli_wallet tool offers two operating modes:
- interactive, when given commands are executed in interactive command line
- daemon based, where it is possible to execute wallet commands using RPC calls. It is important to specify also an IP set, which are allowed to establish connections to the wallet HTTP server. See `--rpc-http-endpoint`, `--daemon`, `--rpc-http-allowip` options for details
To prepare transactions, you need to execute few setup steps in cli_wallet:
- use set_password <password> call establish a password protecting your wallet from unauthorized access
- use unlock <password> to turn full-operation mode in the wallet
- import private key(s) using import_key "WIF-key" command
Keys and password are stored in the wallet.json file.
## Testing
Expand Down
65 changes: 12 additions & 53 deletions doc/exchangequickstart.md
Original file line number Diff line number Diff line change
Expand Up @@ -57,80 +57,39 @@ Both scenarios using --replay-blockchain, allows to continue previously stopped

Script accepts all options provided by hived. If you want to specify different p2p or http port, please use `--p2p-endpoint=` or `--webserver-http-endpoint=` or `--webserver-ws-endpoint=` options, since script will transpose them into docker port mappings accordingly.

### Clone the hive repo
To start a cli_wallet in the dockerized environment, simplest solution is to use [run_cli_wallet_img.sh](://gitlab.syncad.com/hive/hive/-/blob/master/scripts/run_cli_wallet_img.sh). It assumes that wallet.json file should be located in the data-dir mapped to previously started hived container. So using paths specified in the exampole, you should **copy** it into `"$(pwd)/datadir/` directory and next start script.

Pull in the hive repo from the official source on github and then change into the directory that's created for it.
```
git clone https://github.com/openhive-network/hive
cd hive
```

### Build the image from source with docker

Docker isn't just for downloading already built images, it can be used to build from source the same way you would otherwise build. By doing this you ensure that your build environment is identical to what we use to develop the software. Use the below command to start the build:

```
docker build -t=hiveio/hive --target=consensus_node .
```
cli_wallet can be started in the interactive mode (default) or network daemon (handling JSON calls). To start a network daemon let's use `--rpc-http-endpoint=0.0.0.0:8093` and `--rpc-http-allowip=172.17.0.1` (default docker network IP range - please verify it before use).

Don't forget the `.` at the end of the line which indicates the build target is in the current directory.
### Performing each setup steps separately.

### Using our official Docker images without building from source
This kind of setup mostly is covered by general build procedures - described in [building.md](https://gitlab.syncad.com/hive/hive/-/blob/master/doc/building.md#building-under-docker).

If you'd like to use our already pre-built official binary images, it's as simple as downloading it from the Dockerhub registry with only one command:

```
docker pull hiveio/hive
```

Running a docker image, in the API node case (specific to exhange deployment) like also using dockerized version of cli_wallet is described [here](https://gitlab.syncad.com/hive/hive/-/blob/master/README.md#scenarios-of-using-dockerized-hived-assumed-mainnet-configuration)

### Running a binary build without a Docker container

If you build with Docker but do not want to run hived from within a docker container, you can stop here with this step and instead extract the binary from the container with the commands below. If you are going to run hived with docker (recommended method), skip this step altogether. We're simply providing an option for everyone's use-case. Our binaries are built mostly static, only dynamically linking to linux kernel libraries. We have tested and confirmed binaries built in Docker work on Ubuntu and Fedora and will likely work on many other Linux distrubutions. Building the image yourself or pulling one of our pre-built images both work.

To extract the binary you need to start a container and then copy the file from it.
To extract the binary you can use [export-binaries.sh](https://gitlab.syncad.com/hive/hive/-/blob/master/scripts/export-binaries.sh) or start a container and then copy the file from it:

```
docker run -d --name hived-exchange hiveio/hive
docker cp hived-exchange:/usr/local/hive/consensus/bin/hived /local/path/to/hived
docker cp hived-exchange:/usr/local/hive/consensus/bin/cli_wallet /local/path/to/cli_wallet
docker cp hived-exchange:/home/hived/bin/hived /local/path/to/hived
docker cp hived-exchange:/home/hived/bin/cli_wallet /local/path/to/cli_wallet
docker stop hived-exchange
```

### Configuration file

For your convenience, we have provided a provided an [example\_config](example\_config.ini) that we expect should be sufficient to run your exchange node. Be sure to rename it to simply `config.ini`. Be sure to set the account name of your wallet account that you would like to track account history for in the config file. It is defined as `account-history-rocksdb-track-account-range = ["accountname","accountname"]`.
If you want to use custom configuration while using docker, you can place this outside of your container and map to it by adding this argument to your docker run command: `-v /path/to/config.ini:/usr/local/hive/consensus/datadir/config.ini`.

### Create directories to store blockchain and wallet data outside of Docker

For re-usability, you can create directories to store blockchain and wallet data and easily link them inside your docker container.

```
mkdir datadir
mkdir hivewallet
```

### Run the container

The below command will start a daemonized instance opening ports for p2p and RPC while linking the directories we created for blockchain and wallet data inside the container. Fill in `TRACK_ACCOUNT` with the name of your exchange account that you want to follow. The `-v` flags are how you map directories outside of the container to the inside, you list the path to the directories you created earlier before the `:` for each `-v` flag. The restart policy ensures that the container will automatically restart even if your system is restarted.

```
docker run -d --name hived-exchange --env TRACK_ACCOUNT=nameofaccount --env USE_PUBLIC_BLOCKLOG=1 -p 2001:2001 -p 8090:8090 -v /path/to/hivewallet:/var/hivewallet -v /path/to/datadir:/usr/local/hive/consensus/datadir -v /home/exchange/datadir/config.ini:/usr/local/hive/consensus/datadir/config.ini --restart always hiveio/hive
```

You can see that the container is running with the `docker ps` command.

To follow along with the logs, use `docker logs -f`.

Initial syncing will take between 12 and 72 hours depending on your equipment, faster storage devices will take less time and be more efficient. Subsequent restarts will not take as long.

### Running the cli_wallet

The command below will run the cli_wallet from inside the running container while mapping the `wallet.json` to the directory you created for it on the host.

```
docker exec -it hived-exchange /usr/local/hive/consensus/bin/cli_wallet -w /var/hivewallet/wallet.json
```
If you want to use custom configuration while using docker, you can place this outside of your container and copy it into a data-dir mapped to your container.

### Upgrading for major releases that require a full reindex

Expand All @@ -144,7 +103,7 @@ docker rm hived-exchange
rm -rf datadir/blockchain/account-history-rocksdb-storage datadir/blockchain/block_log.index datadir/blockchain/shared_memory.bin
touch datadir/blockchain/force_replay
docker pull hiveio/hive
docker run -d --name hived-exchange --env TRACK_ACCOUNT=nameofaccount --env USE_PUBLIC_BLOCKLOG=1 -p 2001:2001 -p 8090:8090 -v /path/to/hivewallet:/var/hivewallet -v /path/to/datadir:/usr/local/hive/consensus/datadir -v /home/exchange/datadir/config.ini:/usr/local/hive/consensus/datadir/config.ini --restart always hiveio/hive
run_hived_img.sh hiveio/hive --name=hived-exchange --docker-option="-p 8093:8093" --data-dir=$(pwd)/datadir --shared-file-dir=/dev/shm/hived/consensus_node --webserver-http-endpoint=0.0.0.0:8091 --webserver-ws-endpoint=0.0.0.0:8090 --p2p-endpoint=0.0.0.0:2001 --force-replay
```

### Upgrading for releases that do not require a reindex
Expand All @@ -155,5 +114,5 @@ For upgrades that do not require a full replay, you would use the following inst
docker stop hived-exchange
docker rm hived-exchange
docker pull hiveio/hive
docker run -d --name hived-exchange --env TRACK_ACCOUNT=nameofaccount --env USE_PUBLIC_BLOCKLOG=1 -p 2001:2001 -p 8090:8090 -v /path/to/hivewallet:/var/hivewallet -v /path/to/datadir:/usr/local/hive/consensus/datadir -v /home/exchange/datadir/config.ini:/usr/local/hive/consensus/datadir/config.ini --restart always hiveio/hive
run_hived_img.sh hiveio/hive --name=hived-exchange --docker-option="-p 8093:8093" --data-dir=$(pwd)/datadir --shared-file-dir=/dev/shm/hived/consensus_node --webserver-http-endpoint=0.0.0.0:8091 --webserver-ws-endpoint=0.0.0.0:8090 --p2p-endpoint=0.0.0.0:2001
```
2 changes: 1 addition & 1 deletion libraries/wallet/include/hive/wallet/wallet.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -1242,7 +1242,7 @@ class wallet_api

/**
* Account operations have sequence numbers from 0 to N where N is the most recent operation. This method
* returns operations in the range [from-limit, from]
* returns operations in the range (from-limit, from].
*
* @param account - account whose history will be returned
* @param from - the absolute sequence number, -1 means most recent, limit is the number of operations before from.
Expand Down
2 changes: 1 addition & 1 deletion scripts/build_and_setup_exchange_instance.sh
Original file line number Diff line number Diff line change
Expand Up @@ -231,5 +231,5 @@ fi

prepare_data_directory "${HIVED_DATADIR}" ${img_name} ${BLOCK_LOG_BASE_URL}

"${HIVED_SOURCE_DIR}/scripts/run_hived_img.sh" ${img_name} ${IMPLICIT_SHM_DIR} "${RUN_HIVED_IMG_ARGS[@]}"
"${HIVED_SOURCE_DIR}/scripts/run_hived_img.sh" ${img_name} --docker-option="-p 8093:8093" ${IMPLICIT_SHM_DIR} "${RUN_HIVED_IMG_ARGS[@]}"

64 changes: 64 additions & 0 deletions scripts/run_cli_wallet_img.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
#! /bin/bash

SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
echo "$SCRIPTPATH"

LOG_FILE=run_cli_wallet_img.log
source "$SCRIPTPATH/common.sh"

log_exec_params "$@"

print_help () {
echo "Usage: $0 <docker_container_name> [OPTION[=VALUE]]... [<cli_wallet_option>]..."
echo
echo "Allows to start a cli_wallet placed inside started Hive docker container."
echo "OPTIONS:"
echo " --rpc-http-endpoint=<endpoint> Allows to set cli_wallet http endpoint to specified one. This should match to the port specified as cli_wallet port mapping at hived container start (defaults to 8093)"
echo " --rpc-http-allowip=ip_addr Allows to specify the ip allowed to establish connections to the cli_wallet API HTTP server"
echo " --help Display this help screen and exit"
echo
}

CLI_WALLET_ARGS=()

CONTAINER_NAME=instance

add_cli_wallet_arg() {
local arg="$1"
# echo "Processing cli_wallet argument: ${arg}"

CLI_WALLET_ARGS+=("$arg")
}

while [ $# -gt 0 ]; do
case "$1" in
--rpc-http-endpoint=*)
arg="${1#*=}"
add_cli_wallet_arg "--daemon"
add_cli_wallet_arg "--rpc-http-endpoint=${arg}"
;;
--rpc-http-allowip=*)
arg="${1#*=}"
add_cli_wallet_arg "--rpc-http-allowip=${arg}"
;;
-d*)
;; # ignore to avoid duplicated option error
--daemon*)
;; # ignore to avoid duplicated option error

--help)
print_help
exit 0
;;
-*)
add_cli_wallet_arg "$1"
;;
*)
CONTAINER_NAME="${1}"
echo "Using container instance name: $CONTAINER_NAME"
;;
esac
shift
done

docker container exec -it ${CONTAINER_NAME} /home/hived/bin/cli_wallet --wallet-file=/home/hived/datadir/wallet.json "${CLI_WALLET_ARGS[@]}"
2 changes: 1 addition & 1 deletion scripts/run_hived_img.sh
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ while [ $# -gt 0 ]; do
echo "Container name is: $CONTAINER_NAME"
;;
--docker-option=*)
option="${o#*=}"
option="${1#*=}"
add_docker_arg "$option"
;;
--help)
Expand Down
14 changes: 14 additions & 0 deletions tests/api_tests/local_tools.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
import json
from pathlib import Path
from typing import Any, Dict


def read_from_json_pattern(directory: Path, method_name: str) -> Dict[str, Any]:
with open(directory / f'{method_name}.pat.json', 'r') as json_file:
return json.load(json_file)


def write_to_json_pattern(directory: Path, method_name: str, json_response) -> None:
directory.mkdir(parents=True, exist_ok=True)
with open(directory / f'{method_name}.pat.json', 'w') as json_file:
json.dump(json_response, json_file)
Empty file.
Binary file not shown.
Loading

0 comments on commit 80f5c4f

Please sign in to comment.