nightly.yml

name: Nightly -- Full Network Tests

on:
  schedule:
    - cron:  '0 0 * * *'
  workflow_dispatch:

env:
  CARGO_INCREMENTAL: 0 # bookkeeping for incremental builds has overhead, not useful in CI.
  WORKFLOW_URL: https://github.com/maidsafe/stableset_net/actions/runs

jobs:
  e2e:
    name: E2E tests
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]
    steps:
      - uses: actions/checkout@v4

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable

      - uses: Swatinem/rust-cache@v2
        continue-on-error: true
          
      - name: Build binaries
        run: cargo build --release --bin safenode --bin safe --bin faucet
        timeout-minutes: 30

      - name: Start a local network
        uses: maidsafe/sn-local-testnet-action@main
        with:
          action: start
          interval: 2000
          node-path: target/release/safenode
          faucet-path: target/release/faucet
          platform: ${{ matrix.os }}
          build: true

      - name: Check contact peer
        shell: bash
        run: echo "Peer is $SAFE_PEERS"

      - name: Create and fund a wallet to pay for files storage
        run: |
          cargo run --bin faucet --release -- --log-output-dest=data-dir send 1000000 $(cargo run --bin safe --release -- --log-output-dest=data-dir wallet address | tail -n 1) | tail -n 1 > transfer_hex
          cargo run --bin safe --release -- --log-output-dest=data-dir wallet receive --file transfer_hex
        env:
          SN_LOG: "all"
        timeout-minutes: 2

      - name: Start a client to carry out chunk actions
        run: cargo run --bin safe --release -- --log-output-dest=data-dir files upload "./resources" --retry-strategy quick
        env:
          SN_LOG: "all"
        timeout-minutes: 2

      - name: Start a client to create a register
        run: cargo run --bin safe --release -- --log-output-dest=data-dir register create -n baobao
        env:
          SN_LOG: "all"
        timeout-minutes: 2

      - name: Start a client to get a register
        run: cargo run --bin safe --release -- --log-output-dest=data-dir register get -n baobao
        env:
          SN_LOG: "all"
        timeout-minutes: 2

      - name: Start a client to edit a register
        run: cargo run --bin safe --release -- --log-output-dest=data-dir register edit -n baobao wood
        env:
          SN_LOG: "all"
        timeout-minutes: 2

      - name: Stop the local network and upload logs
        if: always()
        uses: maidsafe/sn-local-testnet-action@main
        with:
          action: stop
          log_file_prefix: safe_test_logs_e2e
          platform: ${{ matrix.os }}

      - name: post notification to slack on failure
        if: ${{ failure() }}
        uses: bryannice/gitactions-slack-notification@2.0.0
        env:
          SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }}
          SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}"
          SLACK_TITLE: "Nightly E2E Test Run Failed"

  full_unit:
    name: Full Unit Tests (including proptests)
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]
    steps:
      - uses: actions/checkout@v4

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable

      - uses: Swatinem/rust-cache@v2
        continue-on-error: true

      - name: Build unit tests before running
        run: cargo test --release --lib --bins --no-run 
        timeout-minutes: 30

      - name: Run CLI tests
        timeout-minutes: 25
        run: cargo test --release --package sn_cli

      - name: Run client tests
        timeout-minutes: 25
        run: cargo test --release --package sn_client

      - name: Run network tests
        timeout-minutes: 25
        run: cargo test --release -p sn_networking

      - name: Run protocol tests
        timeout-minutes: 25
        run: cargo test --release -p sn_protocol

      - name: Run transfers tests
        timeout-minutes: 25
        run: cargo test --release --package sn_transfers

      - name: Run register tests
        shell: bash
        timeout-minutes: 50
        env:
          PROPTEST_CASES: 512
        run: cargo test --release -p sn_registers

      - name: post notification to slack on failure
        if: ${{ failure() }}
        uses: bryannice/gitactions-slack-notification@2.0.0
        env:
          SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }}
          SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}"
          SLACK_TITLE: "Nightly Unit Test Run Failed"

  gossipsub:
    if: "!startsWith(github.event.head_commit.message, 'chore(release):')" 
    name: Gossipsub E2E tests
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]
    steps:
      - uses: actions/checkout@v4

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2

      - name: Build binaries
        run: cargo build --release --bin safenode --bin faucet
        timeout-minutes: 30

      - name: Build gossipsub testing executable
        run: cargo test --release -p sn_node --features=local-discovery --test msgs_over_gossipsub --no-run
        env:
          # only set the target dir for windows to bypass the linker issue.
          # happens if we build the node manager via testnet action
          CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }}
        timeout-minutes: 30

      - name: Start a local network
        uses: maidsafe/sn-local-testnet-action@main
        with:
          action: start
          interval: 2000
          node-path: target/release/safenode
          faucet-path: target/release/faucet
          platform: ${{ matrix.os }}
          build: true

      - name: Gossipsub - nodes to subscribe to topics, and publish messages 
        run: cargo test --release -p sn_node --features local-discovery --test msgs_over_gossipsub -- --nocapture
        env:
          CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }}
        timeout-minutes: 20

      - name: Stop the local network and upload logs
        if: always()
        uses: maidsafe/sn-local-testnet-action@main
        with:
          action: stop
          log_file_prefix: safe_test_logs_gossipsub_e2e
          platform: ${{ matrix.os }}

  spend_test:
    name: spend tests against network
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]
    steps:
      - uses: actions/checkout@v4

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable

      - uses: Swatinem/rust-cache@v2
        continue-on-error: true

      - name: Build binaries
        run: cargo build --release --features=local-discovery --bin safenode --bin faucet
        timeout-minutes: 30

      - name: Build testing executable
        run: cargo test --release -p sn_node --features=local-discovery --test sequential_transfers --test storage_payments --no-run
        env:
          # only set the target dir for windows to bypass the linker issue.
          # happens if we build the node manager via testnet action
          CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }}
        timeout-minutes: 30

      - name: Start a local network
        uses: maidsafe/sn-local-testnet-action@main
        with:
          action: start
          interval: 2000
          node-path: target/release/safenode
          faucet-path: target/release/faucet
          platform: ${{ matrix.os }}
          build: true

      - name: execute the spend test
        run: cargo test --release -p sn_node --features="local-discovery" --test sequential_transfers -- --nocapture
        env:
          CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }}
          SN_LOG: "all"
        timeout-minutes: 10

      - name: execute the storage payment tests
        run: cargo test --release -p sn_node --features="local-discovery" --test storage_payments -- --nocapture --test-threads=1          
        env:
          CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }}
          SN_LOG: "all"
        timeout-minutes: 10

      - name: Small wait to allow reward receipt
        run: sleep 30
        timeout-minutes: 1
      
      - name: Stop the local network and upload logs
        if: always()
        uses: maidsafe/sn-local-testnet-action@main
        with:
          action: stop
          log_file_prefix: safe_test_logs_spend
          platform: ${{ matrix.os }}

      - name: post notification to slack on failure
        if: ${{ failure() }}
        uses: bryannice/gitactions-slack-notification@2.0.0
        env:
          SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }}
          SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}"
          SLACK_TITLE: "Nightly Spend Test Run Failed"

  royalty_reward_test:
    name: royalty_reward tests against network
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]
    steps:
      - uses: actions/checkout@v4

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable

      - uses: Swatinem/rust-cache@v2
        continue-on-error: true

      - name: Build binaries
        run: cargo build --release --features=local-discovery,royalties-by-gossip --bin safenode --bin faucet
        timeout-minutes: 30

      - name: Build testing executable
        run: cargo test --release -p sn_node --features=local-discovery,royalties-by-gossip --test nodes_rewards --no-run
        env:
          # only set the target dir for windows to bypass the linker issue.
          # happens if we build the node manager via testnet action
          CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }}
        timeout-minutes: 30

      - name: Start a local network
        uses: maidsafe/sn-local-testnet-action@main
        with:
          action: start
          interval: 2000
          node-path: target/release/safenode
          faucet-path: target/release/faucet
          platform: ${{ matrix.os }}
          build: true

      # This should be first to avoid slow reward acceptance etc
      - name: execute the nodes rewards tests
        run: cargo test --release -p sn_node --features=local-discovery,royalties-by-gossip --test nodes_rewards -- --nocapture --test-threads=1
        env:
          SN_LOG: "all"
          CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }}
        timeout-minutes: 25

      - name: Small wait to allow reward receipt
        run: sleep 30
        timeout-minutes: 1
      
      - name: Stop the local network and upload logs
        if: always()
        uses: maidsafe/sn-local-testnet-action@main
        with:
          action: stop
          log_file_prefix: safe_test_logs_royalty_reward
          platform: ${{ matrix.os }}

      - name: post notification to slack on failure
        if: ${{ failure() }}
        uses: bryannice/gitactions-slack-notification@2.0.0
        env:
          SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }}
          SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}"
          SLACK_TITLE: "Nightly Royalty Reward Test Run Failed"

  token_distribution_test:
    if: "!startsWith(github.event.head_commit.message, 'chore(release):')" 
    name: token distribution test
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]
    steps:
      - uses: actions/checkout@v4

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable

      - uses: Swatinem/rust-cache@v2

      - name: Build binaries
        run: cargo build --release --features=local-discovery,distribution --bin safenode --bin faucet
        timeout-minutes: 30

      - name: Build testing executable
        run: cargo test --release --features=local-discovery,distribution --no-run
        env:
          # only set the target dir for windows to bypass the linker issue.
          # happens if we build the node manager via testnet action
          CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }}
        timeout-minutes: 30

      - name: Start a local network
        uses: maidsafe/sn-local-testnet-action@main
        with:
          action: start
          interval: 2000
          node-path: target/release/safenode
          faucet-path: target/release/faucet
          platform: ${{ matrix.os }}
          build: true

      - name: Check SAFE_PEERS was set
        shell: bash
        run: |
          if [[ -z "$SAFE_PEERS" ]]; then
            echo "The SAFE_PEERS variable has not been set"
            exit 1
          else
            echo "SAFE_PEERS has been set to $SAFE_PEERS"
          fi

      - name: execute token_distribution tests
        run: cargo test --release --features=local-discovery,distribution token_distribution -- --nocapture --test-threads=1
        env:
          SN_LOG: "all"
          CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }}
        timeout-minutes: 25

      - name: Stop the local network and upload logs
        if: always()
        uses: maidsafe/sn-local-testnet-action@main
        with:
          action: stop
          log_file_prefix: safe_test_logs_token_distribution
          platform: ${{ matrix.os }}

  churn:
    name: Network churning tests
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        include:
          - os: ubuntu-latest
            node_data_path: /home/runner/.local/share/safe/node
            safe_path: /home/runner/.local/share/safe
          - os: windows-latest
            node_data_path: C:\\Users\\runneradmin\\AppData\\Roaming\\safe\\node
            safe_path: C:\\Users\\runneradmin\\AppData\\Roaming\\safe
          - os: macos-latest
            node_data_path: /Users/runner/Library/Application Support/safe/node
            safe_path: /Users/runner/Library/Application Support/safe
    steps:
      - uses: actions/checkout@v4

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable

      - uses: Swatinem/rust-cache@v2
        continue-on-error: true

      - name: Build binaries
        run: cargo build --release --features local-discovery --bin safenode --bin faucet
        timeout-minutes: 30

      - name: Build churn tests 
        run: cargo test --release -p sn_node --features=local-discovery --test data_with_churn --no-run
        env:
          # only set the target dir for windows to bypass the linker issue.
          # happens if we build the node manager via testnet action
          CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }}
        timeout-minutes: 30

      - name: Start a local network
        uses: maidsafe/sn-local-testnet-action@main
        with:
          action: start
          interval: 2000
          node-path: target/release/safenode
          faucet-path: target/release/faucet
          platform: ${{ matrix.os }}
          build: true

      - name: Chunks data integrity during nodes churn (during 10min) (in theory)
        run: cargo test --release -p sn_node --features="local-discovery" --test data_with_churn -- --nocapture
        env:
          TEST_DURATION_MINS: 60
          TEST_CHURN_CYCLES: 6
          SN_LOG: "all"
          CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }}
        timeout-minutes: 90
      
      - name: Verify restart of nodes using rg
        shell: bash
        timeout-minutes: 1
        # get the counts, then the specific line, and then the digit count only
        # then check we have an expected level of restarts
        # TODO: make this use an env var, or relate to testnet size
        run : |
          restart_count=$(rg "Node is restarting in" "${{ matrix.node_data_path }}" -c --stats | \
            rg "(\d+) matches" | rg "\d+" -o)
          echo "Restart $restart_count nodes"
          peer_removed=$(rg "PeerRemovedFromRoutingTable" "${{ matrix.node_data_path }}" -c --stats | \
            rg "(\d+) matches" | rg "\d+" -o)
          echo "PeerRemovedFromRoutingTable $peer_removed times"
          if [ $peer_removed -lt $restart_count ]; then
            echo "PeerRemovedFromRoutingTable times of: $peer_removed is less than the restart count of: $restart_count"
            exit 1
          fi
          node_count=$(ls "${{ matrix.node_data_path }}" | wc -l)
          echo "Node dir count is $node_count"
          
        # TODO: reenable this once the testnet dir creation is tidied up to avoid a large count here
        # if [ $restart_count -lt $node_count ]; then
        #   echo "Restart count of: $restart_count is less than the node count of: $node_count"
        #   exit 1
        # fi

      - name: Verify data replication using rg
        shell: bash
        timeout-minutes: 1
        # get the counts, then the specific line, and then the digit count only
        # then check we have an expected level of replication
        # TODO: make this use an env var, or relate to testnet size
        run : |
          fetching_attempt_count=$(rg "FetchingKeysForReplication" "${{ matrix.node_data_path }}" -c --stats | \
            rg "(\d+) matches" | rg "\d+" -o)
          echo "Carried out $fetching_attempt_count fetching attempts"
          node_count=$(ls "${{ matrix.node_data_path }}" | wc -l)
          if [ $fetching_attempt_count -lt $node_count ]; then
            echo "Replication fetching attempts of: $fetching_attempt_count is less than the node count of: $node_count"
            exit 1
          fi

      - name: Stop the local network and upload logs
        if: always()
        uses: maidsafe/sn-local-testnet-action@main
        with:
          action: stop
          log_file_prefix: safe_test_logs_churn
          platform: ${{ matrix.os }}
      
      - name: post notification to slack on failure
        if: ${{ failure() }}
        uses: bryannice/gitactions-slack-notification@2.0.0
        env:
          SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }}
          SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}"
          SLACK_TITLE: "Nightly Churn Test Run Failed"

      # Only error out after uploading the logs
      - name: Don't log raw data
        if: matrix.os != 'windows-latest' # causes error
        shell: bash
        timeout-minutes: 10
        run: |
          if ! rg '^' "${{ matrix.safe_path }}"/*/*/logs | awk 'length($0) > 15000 { print; exit 1 }'
          then
            echo "We are logging an extremely large data"
            exit 1
          fi

  verify_data_location_routing_table:
      name: Verify data location and Routing Table
      runs-on: ${{ matrix.os }}
      strategy:
        matrix:
          include:
            - os: ubuntu-latest
              node_data_path: /home/runner/.local/share/safe/node
              safe_path: /home/runner/.local/share/safe
            - os: windows-latest
              node_data_path: C:\\Users\\runneradmin\\AppData\\Roaming\\safe\\node
              safe_path: C:\\Users\\runneradmin\\AppData\\Roaming\\safe
            - os: macos-latest
              node_data_path: /Users/runner/Library/Application Support/safe/node
              safe_path: /Users/runner/Library/Application Support/safe
      steps:
        - uses: actions/checkout@v4

        - name: Install Rust
          uses: dtolnay/rust-toolchain@stable

        - uses: Swatinem/rust-cache@v2
          continue-on-error: true

        - name: Build binaries
          run: cargo build --release --features local-discovery --bin safenode --bin faucet
          timeout-minutes: 30

        - name: Build data location and routing table tests
          run: cargo test --release -p sn_node --features=local-discovery --test verify_data_location --test verify_routing_table --no-run
          env:
            # only set the target dir for windows to bypass the linker issue.
            # happens if we build the node manager via testnet action
            CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }}
          timeout-minutes: 30

        - name: Start a local network
          uses: maidsafe/sn-local-testnet-action@main
          with:
            action: start
            interval: 2000
            node-path: target/release/safenode
            faucet-path: target/release/faucet
            platform: ${{ matrix.os }}
            build: true

        - name: Verify the Routing table of the nodes
          run: cargo test --release -p sn_node --features="local-discovery" --test verify_routing_table -- --nocapture
          env:
            CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }}
          timeout-minutes: 5

        - name: Verify the location of the data on the network
          run: cargo test --release -p sn_node --features="local-discovery" --test verify_data_location -- --nocapture
          env:
            SN_LOG: "all"
            CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }}
          timeout-minutes: 90

        - name: Verify the routing tables of the nodes
          run: cargo test --release -p sn_node --features="local-discovery" --test verify_routing_table -- --nocapture 
          env:
            CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }}
          timeout-minutes: 5
        
        - name: Verify restart of nodes using rg
          shell: bash
          timeout-minutes: 1
          # get the counts, then the specific line, and then the digit count only
          # then check we have an expected level of restarts
          # TODO: make this use an env var, or relate to testnet size
          run : |
            restart_count=$(rg "Node is restarting in" "${{ matrix.node_data_path }}" -c --stats | \
              rg "(\d+) matches" | rg "\d+" -o)
            echo "Restart $restart_count nodes"
            peer_removed=$(rg "PeerRemovedFromRoutingTable" "${{ matrix.node_data_path }}" -c --stats | \
              rg "(\d+) matches" | rg "\d+" -o)
            echo "PeerRemovedFromRoutingTable $peer_removed times"
            if [ $peer_removed -lt $restart_count ]; then
              echo "PeerRemovedFromRoutingTable times of: $peer_removed is less than the restart count of: $restart_count"
              exit 1
            fi
            node_count=$(ls "${{ matrix.node_data_path }}" | wc -l)
            echo "Node dir count is $node_count"

        - name: Stop the local network and upload logs
          if: always()
          uses: maidsafe/sn-local-testnet-action@main
          with:
            action: stop
            log_file_prefix: safe_test_logs_data_location
            platform: ${{ matrix.os }}
        
        - name: post notification to slack on failure
          if: ${{ failure() }}
          uses: bryannice/gitactions-slack-notification@2.0.0
          env:
            SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }}
            SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}"
            SLACK_TITLE: "Nightly Data Location Test Run Failed"

        # Only error out after uploading the logs
        - name: Don't log raw data
          if: matrix.os != 'windows-latest' # causes error
          shell: bash
          timeout-minutes: 10
          run: |
            if ! rg '^' "${{ matrix.safe_path }}"/*/*/logs | awk 'length($0) > 15000 { print; exit 1 }'
            then
              echo "We are logging an extremely large data"
              exit 1
            fi