test: add zetaclient only upgrade tests

.github/workflows/e2e.yml

@@ -111,6 +111,7 @@ jobs:
     outputs:
       DEFAULT_TESTS: ${{ steps.matrix-conditionals.outputs.DEFAULT_TESTS }}
       UPGRADE_TESTS: ${{ steps.matrix-conditionals.outputs.UPGRADE_TESTS }}
+      UPGRADE_TEST_ZETACLIENT: ${{ steps.matrix-conditionals.outputs.UPGRADE_TEST_ZETACLIENT }}
       CONSENSUS_TESTS: ${{ steps.matrix-conditionals.outputs.CONSENSUS_TESTS }}
       UPGRADE_LIGHT_TESTS: ${{ steps.matrix-conditionals.outputs.UPGRADE_LIGHT_TESTS }}
       UPGRADE_IMPORT_MAINNET_TESTS: ${{ steps.matrix-conditionals.outputs.UPGRADE_IMPORT_MAINNET_TESTS }}
@@ -148,6 +149,7 @@ jobs:
               const labels = await getPrLabels(context.payload.pull_request.number);
               core.setOutput('DEFAULT_TESTS', true);
               core.setOutput('UPGRADE_TESTS', labels.includes('UPGRADE_TESTS'));
+              core.setOutput('UPGRADE_TEST_ZETACLIENT', labels.includes('UPGRADE_TEST_ZETACLIENT'));
               core.setOutput('CONSENSUS_TESTS', labels.includes('CONSENSUS_TESTS'));
               core.setOutput('UPGRADE_LIGHT_TESTS', labels.includes('UPGRADE_LIGHT_TESTS'));
               core.setOutput('UPGRADE_IMPORT_MAINNET_TESTS', labels.includes('UPGRADE_IMPORT_MAINNET_TESTS'));
@@ -168,6 +170,7 @@ jobs:
               // default mergequeue tests
               core.setOutput('DEFAULT_TESTS', true);
               core.setOutput('UPGRADE_TESTS', true);
+              core.setOutput('UPGRADE_TEST_ZETACLIENT', true);
               core.setOutput('ADMIN_UPGRADE_TESTS', true);
 
               // conditional tests based on PR labels
@@ -184,6 +187,7 @@ jobs:
             } else if (context.eventName === 'push' && context.ref.startsWith('refs/heads/release/')) {
               core.setOutput('DEFAULT_TESTS', true);
               core.setOutput('UPGRADE_TESTS', true);
+              core.setOutput('UPGRADE_TEST_ZETACLIENT', true);
               core.setOutput('UPGRADE_LIGHT_TESTS', true);
               core.setOutput('UPGRADE_IMPORT_MAINNET_TESTS', true);
               core.setOutput('ADMIN_TESTS', true);
@@ -197,6 +201,7 @@ jobs:
             } else if (context.eventName === 'schedule') {
               core.setOutput('DEFAULT_TESTS', true);
               core.setOutput('UPGRADE_TESTS', true);
+              core.setOutput('UPGRADE_TEST_ZETACLIENT', true);
               core.setOutput('UPGRADE_LIGHT_TESTS', true);
               core.setOutput('UPGRADE_IMPORT_MAINNET_TESTS', true);
               core.setOutput('ADMIN_TESTS', true);
@@ -213,6 +218,7 @@ jobs:
               const makeTargets = context.payload.inputs['make-targets'].split(',');
               core.setOutput('DEFAULT_TESTS', makeTargets.includes('default-test'));
               core.setOutput('UPGRADE_TESTS', makeTargets.includes('upgrade-test'));
+              core.setOutput('UPGRADE_TEST_ZETACLIENT', makeTargets.includes('upgrade-test-zetaclient'));
               core.setOutput('UPGRADE_LIGHT_TESTS', makeTargets.includes('upgrade-test-light'));
               core.setOutput('UPGRADE_IMPORT_MAINNET_TESTS', makeTargets.includes('upgrade-import-mainnet-test'));
               core.setOutput('ADMIN_TESTS', makeTargets.includes('admin-test'));
@@ -248,6 +254,9 @@ jobs:
           - make-target: "start-upgrade-test"
             runs-on: ubuntu-22.04
             run: ${{ needs.matrix-conditionals.outputs.UPGRADE_TESTS == 'true' }}
+          - make-target: "start-upgrade-test-zetaclient"
+            runs-on: ubuntu-22.04
+            run: ${{ needs.matrix-conditionals.outputs.UPGRADE_TEST_ZETACLIENT == 'true' }}
           - make-target: "start-upgrade-test-light"
             runs-on: ubuntu-22.04
             run: ${{ needs.matrix-conditionals.outputs.UPGRADE_LIGHT_TESTS == 'true' }}

Makefile

@@ -391,6 +391,26 @@ start-upgrade-test-light: zetanode-upgrade
 	export USE_ZETAE2E_ANTE=true && \
 	cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile upgrade -f docker-compose-upgrade.yml up -d
 
+# test zetaclientd-only light upgrade , the test does not wait for the upgrade height , but it is still used to signify that this is a light upgrade[height < 100 == light upgrade]
+start-upgrade-test-zetaclient-light: zetanode-upgrade
+	@echo "--> Starting zetaclientd-only light upgrade test"
+	export LOCALNET_MODE=upgrade && \
+	export UPGRADE_HEIGHT=60 && \
+	export USE_ZETAE2E_ANTE=true && \
+	export E2E_ARGS="--upgrade-contracts" && \
+	export UPGRADE_ZETACLIENT_ONLY=true && \
+	cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile upgrade-zetaclient -f docker-compose-upgrade.yml up -d
+
+# test zetaclientd-only upgrade , the test does not wait for the upgrade height , but it is still used to signify that this is not a light upgrade[height > 100 == regular upgrade]
+start-upgrade-test-zetaclient: zetanode-upgrade solana
+	@echo "--> Starting upgrade test"
+	export LOCALNET_MODE=upgrade && \
+	export UPGRADE_HEIGHT=260 && \
+	export USE_ZETAE2E_ANTE=true && \
+	export E2E_ARGS="--test-solana --test-sui" && \
+	export UPGRADE_ZETACLIENT_ONLY=true && \
+	cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile upgrade-zetaclient --profile solana --profile sui -f docker-compose-upgrade.yml up -d
+
 start-upgrade-test-admin: zetanode-upgrade
 	@echo "--> Starting admin upgrade test"
 	export LOCALNET_MODE=upgrade && \

changelog.md

@@ -55,6 +55,7 @@ by calling `updateAdditionalActionFee` admin function.
 * [4158](https://github.com/zeta-chain/node/pull/4158) - have e2e tests interact with pre-deployed example dApp contract
 * [4165](https://github.com/zeta-chain/node/pull/4165) - fix Sui flaky depositAndCall e2e test in live networks
 * [4177](https://github.com/zeta-chain/node/pull/4177) - add an E2E test to verify depositAndCall with high gas consumption
+* [4310](https://github.com/zeta-chain/node/pull/4310) - add zetaclient only upgrade tests
 
 ## v36.0.0
 

cmd/zetaclientd-supervisor/lib.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"path"
 	"runtime"
+	"strings"
 	"time"
 
 	upgradetypes "cosmossdk.io/x/upgrade/types"
@@ -23,7 +24,15 @@ import (
 
 const zetaclientdBinaryName = "zetaclientd"
 
-var defaultUpgradesDir = os.ExpandEnv("$HOME/.zetaclientd/upgrades")
+var (
+	zetaclientDirectory = os.ExpandEnv("$HOME/.zetaclientd")
+	defaultUpgradesDir  = path.Join(zetaclientDirectory, "upgrades")
+	// defaultZetaclientdBinaryPath is the path where the zetaclientd binary is expected to be found,this is the path used for localnet e2e tests
+	defaultZetaclientdBinaryPath = "/usr/local/bin/zetaclientd"
+	// zetaclientUpgradeTriggerFile is a file watched by the supervisor to trigger a binary upgrade.This should be used when upgrading zetaclient only.
+	// This is primarily used for testing on localnet e2e tests.
+	zetaclientUpgradeTriggerFile = path.Join(zetaclientDirectory, "zetaclientd-upgrade-trigger")
+)
 
 func getLogger(cfg config.Config, out io.Writer) zerolog.Logger {
 	var logger zerolog.Logger
@@ -44,12 +53,13 @@ func getLogger(cfg config.Config, out io.Writer) zerolog.Logger {
 }
 
 type zetaclientdSupervisor struct {
-	zetacoredConn      *grpc.ClientConn
-	reloadSignals      chan bool
-	logger             zerolog.Logger
-	upgradesDir        string
-	upgradePlanName    string
-	enableAutoDownload bool
+	zetacoredConn         *grpc.ClientConn
+	reloadSignals         chan bool
+	logger                zerolog.Logger
+	upgradesDir           string
+	upgradePlanName       string
+	enableAutoDownload    bool
+	zetaclientdBinaryPath string
 }
 
 func newZetaclientdSupervisor(
@@ -66,17 +76,19 @@ func newZetaclientdSupervisor(
 		return nil, fmt.Errorf("grpc dial: %w", err)
 	}
 	return &zetaclientdSupervisor{
-		zetacoredConn:      conn,
-		logger:             logger,
-		reloadSignals:      make(chan bool, 1),
-		upgradesDir:        defaultUpgradesDir,
-		enableAutoDownload: enableAutoDownload,
+		zetacoredConn:         conn,
+		logger:                logger,
+		reloadSignals:         make(chan bool, 1),
+		upgradesDir:           defaultUpgradesDir,
+		enableAutoDownload:    enableAutoDownload,
+		zetaclientdBinaryPath: defaultZetaclientdBinaryPath,
 	}, nil
 }
 
 func (s *zetaclientdSupervisor) Start(ctx context.Context) {
 	go s.watchForVersionChanges(ctx)
 	go s.handleCoreUpgradePlan(ctx)
+	go s.handleFileBasedUpgrade(ctx)
 }
 
 func (s *zetaclientdSupervisor) WaitForReloadSignal(ctx context.Context) {
@@ -239,3 +251,77 @@ func (s *zetaclientdSupervisor) downloadZetaclientd(ctx context.Context, plan *u
 	}
 	return nil
 }
+
+// handleFileBasedUpgrade watches for the trigger file to be created
+// Once created the function updates the zetaclient binary using the URL in the file and triggers a restart
+// This is currently only used for local testing and the trigger file is created by the start-zetae2e.sh script
+func (s *zetaclientdSupervisor) handleFileBasedUpgrade(ctx context.Context) {
+	for {
+		select {
+		case <-time.After(time.Second):
+		case <-ctx.Done():
+			return
+		}
+
+		if _, err := os.Stat(zetaclientUpgradeTriggerFile); err != nil {
+			continue
+		}
+
+		s.logger.Info().Msg("detected file-based upgrade trigger")
+
+		//#nosec G304 used for testing only
+		binURLBytes, err := os.ReadFile(zetaclientUpgradeTriggerFile)
+		if err != nil {
+			panic(fmt.Sprintf("read trigger file: %v", err))
+		}
+		binURL := strings.TrimSpace(string(binURLBytes))
+		if binURL == "" {
+			panic("empty download URL in trigger file")
+		}
+
+		tempDir := os.TempDir()
+		err = s.downloadZetaclientdToPath(ctx, tempDir, binURL)
+		if err != nil {
+			errRemove := os.Remove(tempDir)
+			if errRemove != nil {
+				s.logger.Error().Err(errRemove).Msg("remove temp dir after failed download")
+			}
+			panic(fmt.Sprintf("download zetaclientd: %v", err))
+		}
+
+		err = os.Rename(tempDir, s.zetaclientdBinaryPath)
+		if err != nil {
+			panic(fmt.Sprintf("rename binary into place: %v", err))
+		}
+
+		err = os.Remove(zetaclientUpgradeTriggerFile)
+		if err != nil {
+			panic(fmt.Sprintf("remove trigger file: %v", err))
+		}
+
+		s.reloadSignals <- true
+		s.logger.Info().Msg("zetaclientd binary updated and restart triggered")
+	}
+}
+
+// downloadZetaclientdToPath downloads the zetaclientd binary to the specified path
+func (s *zetaclientdSupervisor) downloadZetaclientdToPath(ctx context.Context, targetPath string, binURL string) error {
+	s.logger.Info().Msgf("downloading zetaclientd to %s from %s", targetPath, binURL)
+
+	err := getter.GetFile(targetPath, binURL, getter.WithContext(ctx), getter.WithUmask(0o750))
+	if err != nil {
+		return fmt.Errorf("get file %s: %w", binURL, err)
+	}
+
+	info, err := os.Stat(targetPath)
+	if err != nil {
+		return fmt.Errorf("stat binary: %w", err)
+	}
+	newMode := info.Mode().Perm() | 0o111
+	err = os.Chmod(targetPath, newMode)
+	if err != nil {
+		return fmt.Errorf("chmod %s: %w", targetPath, err)
+	}
+
+	return nil
+}

cmd/zetae2e/local/local.go

@@ -606,9 +606,14 @@ func localE2ETest(cmd *cobra.Command, _ []string) {
 			os.Exit(1)
 		}
 	}
-	// https://github.com/zeta-chain/node/issues/4038
+
 	// TODO : enable sui gateway upgrade tests to be run multiple times
+	// https://github.com/zeta-chain/node/issues/4038
+	// https://github.com/zeta-chain/node/issues/4315
 	runSuiGatewayUpgradeTests := func() bool {
+		if deployerRunner.IsRunningZetaclientOnlyUpgrade() {
+			return false
+		}
 		// do not if we are running and upgrade and this is the second run
 		if deployerRunner.IsRunningUpgrade() && semver.Major(deployerRunner.GetZetacoredVersion()) == "v0" {
 			return false

contrib/localnet/docker-compose.yml

@@ -326,6 +326,7 @@ services:
       - E2E_ARGS=${E2E_ARGS-}
       - UPGRADE_HEIGHT=${UPGRADE_HEIGHT-}
       - USE_ZETAE2E_ANTE=${USE_ZETAE2E_ANTE-}
+      - UPGRADE_ZETACLIENT_ONLY=${UPGRADE_ZETACLIENT_ONLY-}
       - CI=${CI-}
     volumes:
       - ssh:/root/.ssh
@@ -336,6 +337,7 @@ services:
     hostname: upgrade-host
     profiles:
       - upgrade
+      - upgrade-zetaclient
       - all
     entrypoint: ["/root/start-upgrade-host.sh"]
     networks:
@@ -345,24 +347,24 @@ services:
       - ssh:/root/.ssh
 
   upgrade-orchestrator:
-    # must run from old node for api compatibility
-    image: zetanode:old
-    container_name: upgrade-orchestrator
-    hostname: upgrade-orchestrator
-    profiles:
-      - upgrade
-      - all
-    entrypoint: ["/root/start-upgrade-orchestrator.sh"]
-    networks:
-      mynetwork:
-        ipv4_address: 172.20.0.251
-    depends_on:
-      - zetacore0
-      - upgrade-host
-    environment:
-      - UPGRADE_HEIGHT=${UPGRADE_HEIGHT-}
-    volumes:
-      - ssh:/root/.ssh
+   # must run from old node for api compatibility
+   image: zetanode:old
+   container_name: upgrade-orchestrator
+   hostname: upgrade-orchestrator
+   profiles:
+     - upgrade
+     - all
+   entrypoint: ["/root/start-upgrade-orchestrator.sh"]
+   networks:
+     mynetwork:
+       ipv4_address: 172.20.0.251
+   depends_on:
+     - zetacore0
+     - upgrade-host
+   environment:
+     - UPGRADE_HEIGHT=${UPGRADE_HEIGHT-}
+   volumes:
+     - ssh:/root/.ssh
 
   grafana:
     image: ghcr.io/zeta-chain/grafana-grafana:11.2.0