diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 546737fa2e..3dfa167f3c 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -111,6 +111,7 @@ jobs: outputs: DEFAULT_TESTS: ${{ steps.matrix-conditionals.outputs.DEFAULT_TESTS }} UPGRADE_TESTS: ${{ steps.matrix-conditionals.outputs.UPGRADE_TESTS }} + UPGRADE_TEST_ZETACLIENT: ${{ steps.matrix-conditionals.outputs.UPGRADE_TEST_ZETACLIENT }} CONSENSUS_TESTS: ${{ steps.matrix-conditionals.outputs.CONSENSUS_TESTS }} UPGRADE_LIGHT_TESTS: ${{ steps.matrix-conditionals.outputs.UPGRADE_LIGHT_TESTS }} UPGRADE_IMPORT_MAINNET_TESTS: ${{ steps.matrix-conditionals.outputs.UPGRADE_IMPORT_MAINNET_TESTS }} @@ -148,6 +149,7 @@ jobs: const labels = await getPrLabels(context.payload.pull_request.number); core.setOutput('DEFAULT_TESTS', true); core.setOutput('UPGRADE_TESTS', labels.includes('UPGRADE_TESTS')); + core.setOutput('UPGRADE_TEST_ZETACLIENT', labels.includes('UPGRADE_TEST_ZETACLIENT')); core.setOutput('CONSENSUS_TESTS', labels.includes('CONSENSUS_TESTS')); core.setOutput('UPGRADE_LIGHT_TESTS', labels.includes('UPGRADE_LIGHT_TESTS')); core.setOutput('UPGRADE_IMPORT_MAINNET_TESTS', labels.includes('UPGRADE_IMPORT_MAINNET_TESTS')); @@ -168,6 +170,7 @@ jobs: // default mergequeue tests core.setOutput('DEFAULT_TESTS', true); core.setOutput('UPGRADE_TESTS', true); + core.setOutput('UPGRADE_TEST_ZETACLIENT', true); core.setOutput('ADMIN_UPGRADE_TESTS', true); // conditional tests based on PR labels @@ -184,6 +187,7 @@ jobs: } else if (context.eventName === 'push' && context.ref.startsWith('refs/heads/release/')) { core.setOutput('DEFAULT_TESTS', true); core.setOutput('UPGRADE_TESTS', true); + core.setOutput('UPGRADE_TEST_ZETACLIENT', true); core.setOutput('UPGRADE_LIGHT_TESTS', true); core.setOutput('UPGRADE_IMPORT_MAINNET_TESTS', true); core.setOutput('ADMIN_TESTS', true); @@ -197,6 +201,7 @@ jobs: } else if (context.eventName === 'schedule') { core.setOutput('DEFAULT_TESTS', true); core.setOutput('UPGRADE_TESTS', true); + core.setOutput('UPGRADE_TEST_ZETACLIENT', true); core.setOutput('UPGRADE_LIGHT_TESTS', true); core.setOutput('UPGRADE_IMPORT_MAINNET_TESTS', true); core.setOutput('ADMIN_TESTS', true); @@ -213,6 +218,7 @@ jobs: const makeTargets = context.payload.inputs['make-targets'].split(','); core.setOutput('DEFAULT_TESTS', makeTargets.includes('default-test')); core.setOutput('UPGRADE_TESTS', makeTargets.includes('upgrade-test')); + core.setOutput('UPGRADE_TEST_ZETACLIENT', makeTargets.includes('upgrade-test-zetaclient')); core.setOutput('UPGRADE_LIGHT_TESTS', makeTargets.includes('upgrade-test-light')); core.setOutput('UPGRADE_IMPORT_MAINNET_TESTS', makeTargets.includes('upgrade-import-mainnet-test')); core.setOutput('ADMIN_TESTS', makeTargets.includes('admin-test')); @@ -248,6 +254,9 @@ jobs: - make-target: "start-upgrade-test" runs-on: ubuntu-22.04 run: ${{ needs.matrix-conditionals.outputs.UPGRADE_TESTS == 'true' }} + - make-target: "start-upgrade-test-zetaclient" + runs-on: ubuntu-22.04 + run: ${{ needs.matrix-conditionals.outputs.UPGRADE_TEST_ZETACLIENT == 'true' }} - make-target: "start-upgrade-test-light" runs-on: ubuntu-22.04 run: ${{ needs.matrix-conditionals.outputs.UPGRADE_LIGHT_TESTS == 'true' }} diff --git a/Makefile b/Makefile index a0e49d25a7..5a5785c94b 100644 --- a/Makefile +++ b/Makefile @@ -391,6 +391,26 @@ start-upgrade-test-light: zetanode-upgrade export USE_ZETAE2E_ANTE=true && \ cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile upgrade -f docker-compose-upgrade.yml up -d +# test zetaclientd-only light upgrade , the test does not wait for the upgrade height , but it is still used to signify that this is a light upgrade[height < 100 == light upgrade] +start-upgrade-test-zetaclient-light: zetanode-upgrade + @echo "--> Starting zetaclientd-only light upgrade test" + export LOCALNET_MODE=upgrade && \ + export UPGRADE_HEIGHT=60 && \ + export USE_ZETAE2E_ANTE=true && \ + export E2E_ARGS="--upgrade-contracts" && \ + export UPGRADE_ZETACLIENT_ONLY=true && \ + cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile upgrade-zetaclient -f docker-compose-upgrade.yml up -d + +# test zetaclientd-only upgrade , the test does not wait for the upgrade height , but it is still used to signify that this is not a light upgrade[height > 100 == regular upgrade] +start-upgrade-test-zetaclient: zetanode-upgrade solana + @echo "--> Starting upgrade test" + export LOCALNET_MODE=upgrade && \ + export UPGRADE_HEIGHT=260 && \ + export USE_ZETAE2E_ANTE=true && \ + export E2E_ARGS="--test-solana --test-sui" && \ + export UPGRADE_ZETACLIENT_ONLY=true && \ + cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile upgrade-zetaclient --profile solana --profile sui -f docker-compose-upgrade.yml up -d + start-upgrade-test-admin: zetanode-upgrade @echo "--> Starting admin upgrade test" export LOCALNET_MODE=upgrade && \ diff --git a/changelog.md b/changelog.md index 47b38f62f3..9d348cf693 100644 --- a/changelog.md +++ b/changelog.md @@ -55,6 +55,7 @@ by calling `updateAdditionalActionFee` admin function. * [4158](https://github.com/zeta-chain/node/pull/4158) - have e2e tests interact with pre-deployed example dApp contract * [4165](https://github.com/zeta-chain/node/pull/4165) - fix Sui flaky depositAndCall e2e test in live networks * [4177](https://github.com/zeta-chain/node/pull/4177) - add an E2E test to verify depositAndCall with high gas consumption +* [4310](https://github.com/zeta-chain/node/pull/4310) - add zetaclient only upgrade tests ## v36.0.0 diff --git a/cmd/zetaclientd-supervisor/lib.go b/cmd/zetaclientd-supervisor/lib.go index 0b0ee69b62..a1023e2c37 100644 --- a/cmd/zetaclientd-supervisor/lib.go +++ b/cmd/zetaclientd-supervisor/lib.go @@ -9,6 +9,7 @@ import ( "os" "path" "runtime" + "strings" "time" upgradetypes "cosmossdk.io/x/upgrade/types" @@ -23,7 +24,15 @@ import ( const zetaclientdBinaryName = "zetaclientd" -var defaultUpgradesDir = os.ExpandEnv("$HOME/.zetaclientd/upgrades") +var ( + zetaclientDirectory = os.ExpandEnv("$HOME/.zetaclientd") + defaultUpgradesDir = path.Join(zetaclientDirectory, "upgrades") + // defaultZetaclientdBinaryPath is the path where the zetaclientd binary is expected to be found,this is the path used for localnet e2e tests + defaultZetaclientdBinaryPath = "/usr/local/bin/zetaclientd" + // zetaclientUpgradeTriggerFile is a file watched by the supervisor to trigger a binary upgrade.This should be used when upgrading zetaclient only. + // This is primarily used for testing on localnet e2e tests. + zetaclientUpgradeTriggerFile = path.Join(zetaclientDirectory, "zetaclientd-upgrade-trigger") +) func getLogger(cfg config.Config, out io.Writer) zerolog.Logger { var logger zerolog.Logger @@ -44,12 +53,13 @@ func getLogger(cfg config.Config, out io.Writer) zerolog.Logger { } type zetaclientdSupervisor struct { - zetacoredConn *grpc.ClientConn - reloadSignals chan bool - logger zerolog.Logger - upgradesDir string - upgradePlanName string - enableAutoDownload bool + zetacoredConn *grpc.ClientConn + reloadSignals chan bool + logger zerolog.Logger + upgradesDir string + upgradePlanName string + enableAutoDownload bool + zetaclientdBinaryPath string } func newZetaclientdSupervisor( @@ -66,17 +76,19 @@ func newZetaclientdSupervisor( return nil, fmt.Errorf("grpc dial: %w", err) } return &zetaclientdSupervisor{ - zetacoredConn: conn, - logger: logger, - reloadSignals: make(chan bool, 1), - upgradesDir: defaultUpgradesDir, - enableAutoDownload: enableAutoDownload, + zetacoredConn: conn, + logger: logger, + reloadSignals: make(chan bool, 1), + upgradesDir: defaultUpgradesDir, + enableAutoDownload: enableAutoDownload, + zetaclientdBinaryPath: defaultZetaclientdBinaryPath, }, nil } func (s *zetaclientdSupervisor) Start(ctx context.Context) { go s.watchForVersionChanges(ctx) go s.handleCoreUpgradePlan(ctx) + go s.handleFileBasedUpgrade(ctx) } func (s *zetaclientdSupervisor) WaitForReloadSignal(ctx context.Context) { @@ -239,3 +251,77 @@ func (s *zetaclientdSupervisor) downloadZetaclientd(ctx context.Context, plan *u } return nil } + +// handleFileBasedUpgrade watches for the trigger file to be created +// Once created the function updates the zetaclient binary using the URL in the file and triggers a restart +// This is currently only used for local testing and the trigger file is created by the start-zetae2e.sh script +func (s *zetaclientdSupervisor) handleFileBasedUpgrade(ctx context.Context) { + for { + select { + case <-time.After(time.Second): + case <-ctx.Done(): + return + } + + if _, err := os.Stat(zetaclientUpgradeTriggerFile); err != nil { + continue + } + + s.logger.Info().Msg("detected file-based upgrade trigger") + + //#nosec G304 used for testing only + binURLBytes, err := os.ReadFile(zetaclientUpgradeTriggerFile) + if err != nil { + panic(fmt.Sprintf("read trigger file: %v", err)) + } + binURL := strings.TrimSpace(string(binURLBytes)) + if binURL == "" { + panic("empty download URL in trigger file") + } + + tempDir := os.TempDir() + err = s.downloadZetaclientdToPath(ctx, tempDir, binURL) + if err != nil { + errRemove := os.Remove(tempDir) + if errRemove != nil { + s.logger.Error().Err(errRemove).Msg("remove temp dir after failed download") + } + panic(fmt.Sprintf("download zetaclientd: %v", err)) + } + + err = os.Rename(tempDir, s.zetaclientdBinaryPath) + if err != nil { + panic(fmt.Sprintf("rename binary into place: %v", err)) + } + + err = os.Remove(zetaclientUpgradeTriggerFile) + if err != nil { + panic(fmt.Sprintf("remove trigger file: %v", err)) + } + + s.reloadSignals <- true + s.logger.Info().Msg("zetaclientd binary updated and restart triggered") + } +} + +// downloadZetaclientdToPath downloads the zetaclientd binary to the specified path +func (s *zetaclientdSupervisor) downloadZetaclientdToPath(ctx context.Context, targetPath string, binURL string) error { + s.logger.Info().Msgf("downloading zetaclientd to %s from %s", targetPath, binURL) + + err := getter.GetFile(targetPath, binURL, getter.WithContext(ctx), getter.WithUmask(0o750)) + if err != nil { + return fmt.Errorf("get file %s: %w", binURL, err) + } + + info, err := os.Stat(targetPath) + if err != nil { + return fmt.Errorf("stat binary: %w", err) + } + newMode := info.Mode().Perm() | 0o111 + err = os.Chmod(targetPath, newMode) + if err != nil { + return fmt.Errorf("chmod %s: %w", targetPath, err) + } + + return nil +} diff --git a/cmd/zetae2e/local/local.go b/cmd/zetae2e/local/local.go index a3a8585bb9..fcfc72819a 100644 --- a/cmd/zetae2e/local/local.go +++ b/cmd/zetae2e/local/local.go @@ -606,9 +606,14 @@ func localE2ETest(cmd *cobra.Command, _ []string) { os.Exit(1) } } - // https://github.com/zeta-chain/node/issues/4038 + // TODO : enable sui gateway upgrade tests to be run multiple times + // https://github.com/zeta-chain/node/issues/4038 + // https://github.com/zeta-chain/node/issues/4315 runSuiGatewayUpgradeTests := func() bool { + if deployerRunner.IsRunningZetaclientOnlyUpgrade() { + return false + } // do not if we are running and upgrade and this is the second run if deployerRunner.IsRunningUpgrade() && semver.Major(deployerRunner.GetZetacoredVersion()) == "v0" { return false diff --git a/contrib/localnet/docker-compose.yml b/contrib/localnet/docker-compose.yml index a8074af01c..99622f010e 100644 --- a/contrib/localnet/docker-compose.yml +++ b/contrib/localnet/docker-compose.yml @@ -326,6 +326,7 @@ services: - E2E_ARGS=${E2E_ARGS-} - UPGRADE_HEIGHT=${UPGRADE_HEIGHT-} - USE_ZETAE2E_ANTE=${USE_ZETAE2E_ANTE-} + - UPGRADE_ZETACLIENT_ONLY=${UPGRADE_ZETACLIENT_ONLY-} - CI=${CI-} volumes: - ssh:/root/.ssh @@ -336,6 +337,7 @@ services: hostname: upgrade-host profiles: - upgrade + - upgrade-zetaclient - all entrypoint: ["/root/start-upgrade-host.sh"] networks: @@ -345,24 +347,24 @@ services: - ssh:/root/.ssh upgrade-orchestrator: - # must run from old node for api compatibility - image: zetanode:old - container_name: upgrade-orchestrator - hostname: upgrade-orchestrator - profiles: - - upgrade - - all - entrypoint: ["/root/start-upgrade-orchestrator.sh"] - networks: - mynetwork: - ipv4_address: 172.20.0.251 - depends_on: - - zetacore0 - - upgrade-host - environment: - - UPGRADE_HEIGHT=${UPGRADE_HEIGHT-} - volumes: - - ssh:/root/.ssh + # must run from old node for api compatibility + image: zetanode:old + container_name: upgrade-orchestrator + hostname: upgrade-orchestrator + profiles: + - upgrade + - all + entrypoint: ["/root/start-upgrade-orchestrator.sh"] + networks: + mynetwork: + ipv4_address: 172.20.0.251 + depends_on: + - zetacore0 + - upgrade-host + environment: + - UPGRADE_HEIGHT=${UPGRADE_HEIGHT-} + volumes: + - ssh:/root/.ssh grafana: image: ghcr.io/zeta-chain/grafana-grafana:11.2.0 diff --git a/contrib/localnet/orchestrator/start-zetae2e.sh b/contrib/localnet/orchestrator/start-zetae2e.sh index 25c228cc4d..f327efc6ec 100644 --- a/contrib/localnet/orchestrator/start-zetae2e.sh +++ b/contrib/localnet/orchestrator/start-zetae2e.sh @@ -69,6 +69,23 @@ copy_genesis_file() { fi } +# Create zetaclientd upgrade trigger files on all available zetaclient containers +create_zetaclientd_upgrade_trigger() { + local nodes=("zetaclient0" "zetaclient1" "zetaclient2" "zetaclient3") + local download_url="http://upgrade-host:8000/zetaclientd" + + for node in "${nodes[@]}"; do + # Skip if node is not accessible + ssh -q root@$node "exit" 2>/dev/null || continue + + echo "Creating upgrade trigger file on $node" + ssh root@$node "echo '$download_url' > /root/.zetaclientd/zetaclientd-upgrade-trigger" || continue + echo "Upgrade trigger file created on $node with URL: $download_url" + done + + echo "Zetaclientd upgrade trigger creation completed" +} + get_zetacored_version() { retries=10 node_info="" @@ -291,6 +308,7 @@ if [ "$LOCALNET_MODE" == "upgrade" ]; then # shellcheck disable=SC2155 COMMON_ARGS="--skip-header-proof --skip-tracker-check" USE_ZETAE2E_ANTE=${USE_ZETAE2E_ANTE:=false} + UPGRADE_ZETACLIENT_ONLY=${UPGRADE_ZETACLIENT_ONLY:=false} # if enabled, fetches zetae2e binary from the previous version # ante means "before" in Latin (used in Cosmos terminology) @@ -319,39 +337,40 @@ if [ "$LOCALNET_MODE" == "upgrade" ]; then # Run zetae2e, if the upgrade height is greater than 100 to populate the state if [ "$UPGRADE_HEIGHT" -gt 100 ]; then echo "Running E2E command to setup the networks and populate the state..." + # Use light flag to ensure tests can complete before the upgrade height + zetae2e-ante local $E2E_ARGS --skip-setup --config "$deployed_config_path" --light ${COMMON_ARGS} + if [ $? -ne 0 ]; then + echo "First E2E failed" + exit 1 + fi + fi - # Use light flag to ensure tests can complete before the upgrade height - # skip-bitcoin-dust-withdraw flag can be removed after v23 is released - zetae2e-ante local $E2E_ARGS --skip-setup --config "$deployed_config_path" --light ${COMMON_ARGS} - if [ $? -ne 0 ]; then - echo "First E2E failed" - exit 1 + # If this is a zetaclient only upgrade , update the binary and proceed , if not wait for the upgrade height + if [ "$UPGRADE_ZETACLIENT_ONLY" = true ]; then + echo "Zetaclientd-only upgrade mode: updating zetaclientd to $NEW_VERSION" + create_zetaclientd_upgrade_trigger + else + echo "Waiting for upgrade height..." + CURRENT_HEIGHT=0 + WAIT_HEIGHT=$(( UPGRADE_HEIGHT - 1 )) + while [[ $CURRENT_HEIGHT -lt $WAIT_HEIGHT ]] + do + CURRENT_HEIGHT=$(curl -s zetacore0:26657/status | jq -r '.result.sync_info.latest_block_height') + echo Current height is "$CURRENT_HEIGHT", waiting for "$WAIT_HEIGHT" + sleep 2 + done + + echo "Waiting 10 seconds for node to restart..." + sleep 10 + if [[ "$OLD_VERSION" == "$NEW_VERSION" ]]; then + echo "Version did not change after upgrade height, maybe the upgrade did not run?" + exit 2 fi fi - echo "Waiting for upgrade height..." - CURRENT_HEIGHT=0 - WAIT_HEIGHT=$(( UPGRADE_HEIGHT - 1 )) - # wait for upgrade height - while [[ $CURRENT_HEIGHT -lt $WAIT_HEIGHT ]] - do - CURRENT_HEIGHT=$(curl -s zetacore0:26657/status | jq -r '.result.sync_info.latest_block_height') - echo Current height is "$CURRENT_HEIGHT", waiting for "$WAIT_HEIGHT" - sleep 2 - done - - echo "Waiting 10 seconds for node to restart..." - sleep 10 - NEW_VERSION=$(get_zetacored_version) - echo "Upgrade result: ${OLD_VERSION} -> ${NEW_VERSION}" - if [[ "$OLD_VERSION" == "$NEW_VERSION" ]]; then - echo "Version did not change after upgrade height, maybe the upgrade did not run?" - exit 2 - fi - # wait for zevm endpoint to come up sleep 10 echo "Running E2E command to test the network after upgrade..." diff --git a/e2e/e2etests/e2etests.go b/e2e/e2etests/e2etests.go index 63a9b7e269..09400eabe9 100644 --- a/e2e/e2etests/e2etests.go +++ b/e2e/e2etests/e2etests.go @@ -1364,6 +1364,7 @@ var AllE2ETests = []runner.E2ETest{ "bitcoin -> zevm call revert with excessive funds", []runner.ArgDefinition{}, TestBitcoinToZEVMCallExcessiveFundsRevert, + runner.WithMinimumVersion("v37.0.0"), ), runner.NewE2ETest( TestBitcoinDepositAndAbortWithLowDepositFeeName, diff --git a/e2e/runner/runner.go b/e2e/runner/runner.go index 0941de99cc..79df0f43f1 100644 --- a/e2e/runner/runner.go +++ b/e2e/runner/runner.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "regexp" + "strconv" "sync" "time" @@ -59,7 +60,8 @@ type E2ERunnerOption func(*E2ERunner) // Important ENV const ( - EnvKeyLocalnetMode = "LOCALNET_MODE" + EnvKeyLocalnetMode = "LOCALNET_MODE" + EnvKeyUpgradeZetaclientOnly = "UPGRADE_ZETACLIENT_ONLY" LocalnetModeUpgrade = "upgrade" LocalNetModeTSSMigration = "tss-migration" @@ -495,6 +497,15 @@ func (r *E2ERunner) IsRunningTssMigration() bool { return os.Getenv(EnvKeyLocalnetMode) == LocalnetModeUpgrade } +func (r *E2ERunner) IsRunningZetaclientOnlyUpgrade() bool { + value := os.Getenv(EnvKeyUpgradeZetaclientOnly) + enabled, err := strconv.ParseBool(value) + if err != nil { + return false + } + return enabled +} + func (r *E2ERunner) IsRunningUpgradeOrTSSMigration() bool { return os.Getenv(EnvKeyLocalnetMode) == LocalnetModeUpgrade || os.Getenv(EnvKeyLocalnetMode) == LocalNetModeTSSMigration diff --git a/e2e/runner/setup_sui.go b/e2e/runner/setup_sui.go index 7bd3ef4b57..a10ff8305b 100644 --- a/e2e/runner/setup_sui.go +++ b/e2e/runner/setup_sui.go @@ -520,7 +520,7 @@ func (r *E2ERunner) setSuiChainParams(resetNonces bool) error { }, } if err := r.ZetaTxServer.UpdateChainParams(chainParams); err != nil { - return errors.Wrap(err, "unable to broadcast solana chain params tx") + return errors.Wrap(err, "unable to broadcast sui chain params tx") } if resetNonces { diff --git a/e2e/runner/upgrade.go b/e2e/runner/upgrade.go index 666524e751..75faabe393 100644 --- a/e2e/runner/upgrade.go +++ b/e2e/runner/upgrade.go @@ -129,14 +129,16 @@ func (r *E2ERunner) AddPreUpgradeHandler(upgradeFrom string, preHandler func()) // AddPostUpgradeHandler adds a handler to run any logic after and upgrade to enable tests to be executed // Note This is handler is not related to the cosmos-sdk upgrade handler in any way func (r *E2ERunner) AddPostUpgradeHandler(upgradeFrom string, postHandler func()) { - version := r.GetZetacoredVersion() - versionMajorIsZero := semver.Major(version) == "v0" - oldVersion := fmt.Sprintf("v%s", os.Getenv("OLD_VERSION")) - - // Run the handler only if this is the second run of the upgrade tests - if !r.IsRunningUpgrade() || !r.IsRunningTssMigration() || !versionMajorIsZero || - checkVersion(upgradeFrom, oldVersion) { - return + if !r.IsRunningZetaclientOnlyUpgrade() { + version := r.GetZetacoredVersion() + versionMajorIsZero := semver.Major(version) == "v0" + oldVersion := fmt.Sprintf("v%s", os.Getenv("OLD_VERSION")) + + // Run the handler only if this is the second run of the upgrade tests + if !r.IsRunningUpgrade() || !r.IsRunningTssMigration() || !versionMajorIsZero || + checkVersion(upgradeFrom, oldVersion) { + return + } } postHandler()