test: add zetaclient only upgrade tests

.github/workflows/e2e.yml

@@ -111,6 +111,7 @@ jobs:
     outputs:
       DEFAULT_TESTS: ${{ steps.matrix-conditionals.outputs.DEFAULT_TESTS }}
       UPGRADE_TESTS: ${{ steps.matrix-conditionals.outputs.UPGRADE_TESTS }}
+      UPGRADE_TEST_ZETACLIENT: ${{ steps.matrix-conditionals.outputs.UPGRADE_TEST_ZETACLIENT }}
       CONSENSUS_TESTS: ${{ steps.matrix-conditionals.outputs.CONSENSUS_TESTS }}
       UPGRADE_LIGHT_TESTS: ${{ steps.matrix-conditionals.outputs.UPGRADE_LIGHT_TESTS }}
       UPGRADE_IMPORT_MAINNET_TESTS: ${{ steps.matrix-conditionals.outputs.UPGRADE_IMPORT_MAINNET_TESTS }}
@@ -148,6 +149,7 @@ jobs:
               const labels = await getPrLabels(context.payload.pull_request.number);
               core.setOutput('DEFAULT_TESTS', true);
               core.setOutput('UPGRADE_TESTS', labels.includes('UPGRADE_TESTS'));
+              core.setOutput('UPGRADE_TEST_ZETACLIENT', labels.includes('UPGRADE_TEST_ZETACLIENT'));
               core.setOutput('CONSENSUS_TESTS', labels.includes('CONSENSUS_TESTS'));
               core.setOutput('UPGRADE_LIGHT_TESTS', labels.includes('UPGRADE_LIGHT_TESTS'));
               core.setOutput('UPGRADE_IMPORT_MAINNET_TESTS', labels.includes('UPGRADE_IMPORT_MAINNET_TESTS'));
@@ -168,6 +170,7 @@ jobs:
               // default mergequeue tests
               core.setOutput('DEFAULT_TESTS', true);
               core.setOutput('UPGRADE_TESTS', true);
+              core.setOutput('UPGRADE_TEST_ZETACLIENT', true);
               core.setOutput('ADMIN_UPGRADE_TESTS', true);
 
               // conditional tests based on PR labels
@@ -184,6 +187,7 @@ jobs:
             } else if (context.eventName === 'push' && context.ref.startsWith('refs/heads/release/')) {
               core.setOutput('DEFAULT_TESTS', true);
               core.setOutput('UPGRADE_TESTS', true);
+              core.setOutput('UPGRADE_TEST_ZETACLIENT', true);
               core.setOutput('UPGRADE_LIGHT_TESTS', true);
               core.setOutput('UPGRADE_IMPORT_MAINNET_TESTS', true);
               core.setOutput('ADMIN_TESTS', true);
@@ -197,6 +201,7 @@ jobs:
             } else if (context.eventName === 'schedule') {
               core.setOutput('DEFAULT_TESTS', true);
               core.setOutput('UPGRADE_TESTS', true);
+              core.setOutput('UPGRADE_TEST_ZETACLIENT', true);
               core.setOutput('UPGRADE_LIGHT_TESTS', true);
               core.setOutput('UPGRADE_IMPORT_MAINNET_TESTS', true);
               core.setOutput('ADMIN_TESTS', true);
@@ -213,6 +218,7 @@ jobs:
               const makeTargets = context.payload.inputs['make-targets'].split(',');
               core.setOutput('DEFAULT_TESTS', makeTargets.includes('default-test'));
               core.setOutput('UPGRADE_TESTS', makeTargets.includes('upgrade-test'));
+              core.setOutput('UPGRADE_TEST_ZETACLIENT', makeTargets.includes('upgrade-test-zetaclient'));
               core.setOutput('UPGRADE_LIGHT_TESTS', makeTargets.includes('upgrade-test-light'));
               core.setOutput('UPGRADE_IMPORT_MAINNET_TESTS', makeTargets.includes('upgrade-import-mainnet-test'));
               core.setOutput('ADMIN_TESTS', makeTargets.includes('admin-test'));
@@ -248,6 +254,9 @@ jobs:
           - make-target: "start-upgrade-test"
             runs-on: ubuntu-22.04
             run: ${{ needs.matrix-conditionals.outputs.UPGRADE_TESTS == 'true' }}
+          - make-target: "start-upgrade-test-zetaclient"
+            runs-on: ubuntu-22.04
+            run: ${{ needs.matrix-conditionals.outputs.UPGRADE_TEST_ZETACLIENT == 'true' }}
           - make-target: "start-upgrade-test-light"
             runs-on: ubuntu-22.04
             run: ${{ needs.matrix-conditionals.outputs.UPGRADE_LIGHT_TESTS == 'true' }}

Makefile

@@ -391,6 +391,23 @@ start-upgrade-test-light: zetanode-upgrade
 	export USE_ZETAE2E_ANTE=true && \
 	cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile upgrade -f docker-compose-upgrade.yml up -d
 
+start-upgrade-test-zetaclient-light: zetanode-upgrade
+	@echo "--> Starting zetaclientd-only light upgrade test"
+	export LOCALNET_MODE=upgrade && \
+	export UPGRADE_HEIGHT=60 && \
+	export USE_ZETAE2E_ANTE=true && \
+	export UPGRADE_ZETACLIENT_ONLY=true && \
+	cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile upgrade-zetaclient -f docker-compose-upgrade.yml up -d
+
+start-upgrade-test-zetaclient: zetanode-upgrade solana
+	@echo "--> Starting upgrade test"
+	export LOCALNET_MODE=upgrade && \
+	export UPGRADE_HEIGHT=260 && \
+	export USE_ZETAE2E_ANTE=true && \
+	export E2E_ARGS="--test-solana --test-sui" && \
+	export UPGRADE_ZETACLIENT_ONLY=true && \
+	cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile upgrade-zetaclient --profile solana --profile sui -f docker-compose-upgrade.yml up -d
+
 start-upgrade-test-admin: zetanode-upgrade
 	@echo "--> Starting admin upgrade test"
 	export LOCALNET_MODE=upgrade && \

changelog.md

@@ -55,6 +55,7 @@ by calling `updateAdditionalActionFee` admin function.
 * [4158](https://github.com/zeta-chain/node/pull/4158) - have e2e tests interact with pre-deployed example dApp contract
 * [4165](https://github.com/zeta-chain/node/pull/4165) - fix Sui flaky depositAndCall e2e test in live networks
 * [4177](https://github.com/zeta-chain/node/pull/4177) - add an E2E test to verify depositAndCall with high gas consumption
+* [4310](https://github.com/zeta-chain/node/pull/4310) - add zetaclient only upgrade tests
 
 ## v36.0.0
 

cmd/zetaclientd-supervisor/lib.go

@@ -77,6 +77,7 @@ func newZetaclientdSupervisor(
 func (s *zetaclientdSupervisor) Start(ctx context.Context) {
 	go s.watchForVersionChanges(ctx)
 	go s.handleCoreUpgradePlan(ctx)
+	go s.handleFileBasedUpgrade(ctx)
 }
 
 func (s *zetaclientdSupervisor) WaitForReloadSignal(ctx context.Context) {
@@ -239,3 +240,69 @@ func (s *zetaclientdSupervisor) downloadZetaclientd(ctx context.Context, plan *u
 	}
 	return nil
 }
+
+// handleFileBasedUpgrade watches for a specific file to appear to trigger an upgrade
+func (s *zetaclientdSupervisor) handleFileBasedUpgrade(ctx context.Context) {
+	triggerFile := "/root/.zetaclientd/zetaclientd-upgrade-trigger"
+
+	for {
+		select {
+		case <-time.After(time.Second):
+		case <-ctx.Done():
+			return
+		}
+
+		if _, err := os.Stat(triggerFile); err != nil {
+			continue
+		}
+
+		s.logger.Info().Msg("detected file-based upgrade trigger")
+
+		tempPath := "/tmp/zetaclientd.new"
+		err := s.downloadZetaclientdToPath(ctx, tempPath)
+		if err != nil {
+			s.logger.Error().Err(err).Msg("failed to download zetaclientd binary")
+			continue
+		}
+
+		s.logger.Info().Msg("killing existing zetaclientd process")
+		err = os.Rename(tempPath, "/usr/local/bin/zetaclientd")
+		if err != nil {
+			s.logger.Error().Err(err).Msg("failed to replace zetaclientd binary")
+			continue
+		}
+
+		// Remove trigger file
+		err = os.Remove(triggerFile)
+		if err != nil {
+			panic(fmt.Sprintf("remove trigger file: %v", err))
+		}
+
+		s.reloadSignals <- true
+		s.logger.Info().Msg("zetaclientd binary updated and restart triggered")
+	}
+}
+
+// downloadZetaclientdToPath downloads the zetaclientd binary to the specified path
+func (s *zetaclientdSupervisor) downloadZetaclientdToPath(ctx context.Context, targetPath string) error {
+	binURL := "http://upgrade-host:8000/zetaclientd"
+
+	s.logger.Info().Msgf("downloading zetaclientd to %s", targetPath)
+
+	err := getter.GetFile(targetPath, binURL, getter.WithContext(ctx), getter.WithUmask(0o750))
+	if err != nil {
+		return fmt.Errorf("get file %s: %w", binURL, err)
+	}
+
+	info, err := os.Stat(targetPath)
+	if err != nil {
+		return fmt.Errorf("stat binary: %w", err)
+	}
+	newMode := info.Mode().Perm() | 0o111
+	err = os.Chmod(targetPath, newMode)
+	if err != nil {
+		return fmt.Errorf("chmod %s: %w", targetPath, err)
+	}
+
+	return nil
+}

cmd/zetae2e/local/local.go

@@ -609,6 +609,9 @@ func localE2ETest(cmd *cobra.Command, _ []string) {
 	// https://github.com/zeta-chain/node/issues/4038
 	// TODO : enable sui gateway upgrade tests to be run multiple times
 	runSuiGatewayUpgradeTests := func() bool {
+		if deployerRunner.IsRunningZetaclientOnlyUpgrade() {
+			return false
+		}
 		// do not if we are running and upgrade and this is the second run
 		if deployerRunner.IsRunningUpgrade() && semver.Major(deployerRunner.GetZetacoredVersion()) == "v0" {
 			return false

contrib/localnet/docker-compose.yml

@@ -326,6 +326,7 @@ services:
       - E2E_ARGS=${E2E_ARGS-}
       - UPGRADE_HEIGHT=${UPGRADE_HEIGHT-}
       - USE_ZETAE2E_ANTE=${USE_ZETAE2E_ANTE-}
+      - UPGRADE_ZETACLIENT_ONLY=${UPGRADE_ZETACLIENT_ONLY-}
       - CI=${CI-}
     volumes:
       - ssh:/root/.ssh
@@ -336,6 +337,7 @@ services:
     hostname: upgrade-host
     profiles:
       - upgrade
+      - upgrade-zetaclient
       - all
     entrypoint: ["/root/start-upgrade-host.sh"]
     networks:
@@ -345,24 +347,24 @@ services:
       - ssh:/root/.ssh
 
   upgrade-orchestrator:
-    # must run from old node for api compatibility
-    image: zetanode:old
-    container_name: upgrade-orchestrator
-    hostname: upgrade-orchestrator
-    profiles:
-      - upgrade
-      - all
-    entrypoint: ["/root/start-upgrade-orchestrator.sh"]
-    networks:
-      mynetwork:
-        ipv4_address: 172.20.0.251
-    depends_on:
-      - zetacore0
-      - upgrade-host
-    environment:
-      - UPGRADE_HEIGHT=${UPGRADE_HEIGHT-}
-    volumes:
-      - ssh:/root/.ssh
+   # must run from old node for api compatibility
+   image: zetanode:old
+   container_name: upgrade-orchestrator
+   hostname: upgrade-orchestrator
+   profiles:
+     - upgrade
+     - all
+   entrypoint: ["/root/start-upgrade-orchestrator.sh"]
+   networks:
+     mynetwork:
+       ipv4_address: 172.20.0.251
+   depends_on:
+     - zetacore0
+     - upgrade-host
+   environment:
+     - UPGRADE_HEIGHT=${UPGRADE_HEIGHT-}
+   volumes:
+     - ssh:/root/.ssh
 
   grafana:
     image: ghcr.io/zeta-chain/grafana-grafana:11.2.0

contrib/localnet/orchestrator/start-zetae2e.sh

@@ -69,6 +69,22 @@ copy_genesis_file() {
   fi
 }
 
+# Create zetaclientd upgrade trigger files on all available zetaclient containers
+create_zetaclientd_upgrade_trigger() {
+  local nodes=("zetaclient0" "zetaclient1" "zetaclient2" "zetaclient3")
+
+  for node in "${nodes[@]}"; do
+    # Skip if node is not accessible
+    ssh -q root@$node "exit" 2>/dev/null || continue
+
+    echo "Creating upgrade trigger file on $node"
+    ssh root@$node "touch /root/.zetaclientd/zetaclientd-upgrade-trigger" || continue
+    echo "Upgrade trigger file created on $node"
+  done
+
+  echo "Zetaclientd upgrade trigger creation completed"
+}
+
 get_zetacored_version() {
   retries=10
   node_info=""
@@ -291,6 +307,7 @@ if [ "$LOCALNET_MODE" == "upgrade" ]; then
   # shellcheck disable=SC2155
   COMMON_ARGS="--skip-header-proof --skip-tracker-check"
   USE_ZETAE2E_ANTE=${USE_ZETAE2E_ANTE:=false}
+  UPGRADE_ZETACLIENT_ONLY=${UPGRADE_ZETACLIENT_ONLY:=false}
 
   # if enabled, fetches zetae2e binary from the previous version
   # ante means "before" in Latin (used in Cosmos terminology)
@@ -319,14 +336,13 @@ if [ "$LOCALNET_MODE" == "upgrade" ]; then
   # Run zetae2e, if the upgrade height is greater than 100 to populate the state
   if [ "$UPGRADE_HEIGHT" -gt 100 ]; then
     echo "Running E2E command to setup the networks and populate the state..."
-
-    # Use light flag to ensure tests can complete before the upgrade height
-    # skip-bitcoin-dust-withdraw flag can be removed after v23 is released
-    zetae2e-ante local $E2E_ARGS --skip-setup --config "$deployed_config_path" --light ${COMMON_ARGS}
-    if [ $? -ne 0 ]; then
-      echo "First E2E failed"
-      exit 1
-    fi
+      # Use light flag to ensure tests can complete before the upgrade height
+      # skip-bitcoin-dust-withdraw flag can be removed after v23 is released
+      zetae2e-ante local $E2E_ARGS --skip-setup --config "$deployed_config_path" --light ${COMMON_ARGS}
+      if [ $? -ne 0 ]; then
+        echo "First E2E failed"
+        exit 1
+      fi
   fi
 
   echo "Waiting for upgrade height..."
@@ -347,9 +363,14 @@ if [ "$LOCALNET_MODE" == "upgrade" ]; then
 
   echo "Upgrade result: ${OLD_VERSION} -> ${NEW_VERSION}"
 
-  if [[ "$OLD_VERSION" == "$NEW_VERSION" ]]; then
-    echo "Version did not change after upgrade height, maybe the upgrade did not run?"
-    exit 2
+  if [ "$UPGRADE_ZETACLIENT_ONLY" = true ]; then
+    echo "Zetaclientd-only upgrade mode: updating zetaclientd to $NEW_VERSION"
+    create_zetaclientd_upgrade_trigger
+  else
+    if [[ "$OLD_VERSION" == "$NEW_VERSION" ]]; then
+      echo "Version did not change after upgrade height, maybe the upgrade did not run?"
+      exit 2
+    fi
   fi
 
   # wait for zevm endpoint to come up

e2e/e2etests/e2etests.go

@@ -1364,6 +1364,7 @@ var AllE2ETests = []runner.E2ETest{
 		"bitcoin -> zevm call revert with excessive funds",
 		[]runner.ArgDefinition{},
 		TestBitcoinToZEVMCallExcessiveFundsRevert,
+		runner.WithMinimumVersion("v37.0.0"),
 	),
 	runner.NewE2ETest(
 		TestBitcoinDepositAndAbortWithLowDepositFeeName,

e2e/runner/runner.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"strconv"
 	"sync"
 	"time"
 
@@ -59,7 +60,8 @@ type E2ERunnerOption func(*E2ERunner)
 
 // Important ENV
 const (
-	EnvKeyLocalnetMode = "LOCALNET_MODE"
+	EnvKeyLocalnetMode          = "LOCALNET_MODE"
+	EnvKeyUpgradeZetaclientOnly = "UPGRADE_ZETACLIENT_ONLY"
 
 	LocalnetModeUpgrade      = "upgrade"
 	LocalNetModeTSSMigration = "tss-migration"
@@ -495,6 +497,15 @@ func (r *E2ERunner) IsRunningTssMigration() bool {
 	return os.Getenv(EnvKeyLocalnetMode) == LocalnetModeUpgrade
 }
 
+func (r *E2ERunner) IsRunningZetaclientOnlyUpgrade() bool {
+	value := os.Getenv(EnvKeyUpgradeZetaclientOnly)
+	enabled, err := strconv.ParseBool(value)
+	if err != nil {
+		return false
+	}
+	return enabled
+}
+
 func (r *E2ERunner) IsRunningUpgradeOrTSSMigration() bool {
 	return os.Getenv(EnvKeyLocalnetMode) == LocalnetModeUpgrade ||
 		os.Getenv(EnvKeyLocalnetMode) == LocalNetModeTSSMigration

e2e/runner/setup_sui.go

@@ -520,7 +520,7 @@ func (r *E2ERunner) setSuiChainParams(resetNonces bool) error {
 		},
 	}
 	if err := r.ZetaTxServer.UpdateChainParams(chainParams); err != nil {
-		return errors.Wrap(err, "unable to broadcast solana chain params tx")
+		return errors.Wrap(err, "unable to broadcast sui chain params tx")
 	}
 
 	if resetNonces {

e2e/runner/upgrade.go

@@ -129,14 +129,16 @@ func (r *E2ERunner) AddPreUpgradeHandler(upgradeFrom string, preHandler func())
 // AddPostUpgradeHandler adds a handler to run any logic after and upgrade to enable tests to be executed
 // Note This is handler is not related to the cosmos-sdk upgrade handler in any way
 func (r *E2ERunner) AddPostUpgradeHandler(upgradeFrom string, postHandler func()) {
-	version := r.GetZetacoredVersion()
-	versionMajorIsZero := semver.Major(version) == "v0"
-	oldVersion := fmt.Sprintf("v%s", os.Getenv("OLD_VERSION"))
-
-	// Run the handler only if this is the second run of the upgrade tests
-	if !r.IsRunningUpgrade() || !r.IsRunningTssMigration() || !versionMajorIsZero ||
-		checkVersion(upgradeFrom, oldVersion) {
-		return
+	if !r.IsRunningZetaclientOnlyUpgrade() {
+		version := r.GetZetacoredVersion()
+		versionMajorIsZero := semver.Major(version) == "v0"
+		oldVersion := fmt.Sprintf("v%s", os.Getenv("OLD_VERSION"))
+
+		// Run the handler only if this is the second run of the upgrade tests
+		if !r.IsRunningUpgrade() || !r.IsRunningTssMigration() || !versionMajorIsZero ||
+			checkVersion(upgradeFrom, oldVersion) {
+			return
+		}
 	}
 
 	postHandler()