From 7afa9272f4b875b0fd6f4a785e2c489bdc0cee35 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Tue, 21 Jan 2025 22:24:15 +0000 Subject: [PATCH] Complete partial downgraded cluster cancellation Signed-off-by: Chun-Hung Tseng --- tests/e2e/cluster_downgrade_test.go | 55 +++++++++++++++++++++-------- tests/framework/e2e/downgrade.go | 8 ++--- 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/tests/e2e/cluster_downgrade_test.go b/tests/e2e/cluster_downgrade_test.go index ffab9fba7ab..7cbc03fe653 100644 --- a/tests/e2e/cluster_downgrade_test.go +++ b/tests/e2e/cluster_downgrade_test.go @@ -45,7 +45,7 @@ const ( noCancellation CancellationState = iota cancelRightBeforeEnable cancelRightAfterEnable - cancelAfterDowngradingSome + cancelAfterDowngrading ) func TestDowngradeUpgradeClusterOf1(t *testing.T) { @@ -80,13 +80,13 @@ func TestDowngradeCancellationRightAfterEnablingClusterOf3(t *testing.T) { testDowngradeUpgrade(t, 3, 3, false, cancelRightAfterEnable) } -// func TestDowngradeCancellationAfterDowngrading1InClusterOf3(t *testing.T) { -// testDowngradeUpgrade(t, 1, 3, false, cancelAfterDowngradingSome) -// } +func TestDowngradeCancellationAfterDowngrading1InClusterOf3(t *testing.T) { + testDowngradeUpgrade(t, 1, 3, false, cancelAfterDowngrading) +} -// func TestDowngradeCancellationAfterDowngrading2InClusterOf3(t *testing.T) { -// testDowngradeUpgrade(t, 2, 3, false, cancelAfterDowngradingSome) -// } +func TestDowngradeCancellationAfterDowngrading2InClusterOf3(t *testing.T) { + testDowngradeUpgrade(t, 2, 3, false, cancelAfterDowngrading) +} func testDowngradeUpgrade(t *testing.T, cancellationSize int, clusterSize int, triggerSnapshot bool, triggerCancellation CancellationState) { currentEtcdBinary := e2e.BinPath.Etcd @@ -144,23 +144,25 @@ func testDowngradeUpgrade(t *testing.T, cancellationSize int, clusterSize int, t if triggerCancellation == cancelRightBeforeEnable { t.Logf("Cancelling downgrade before enabling") - e2e.DowngradeCancel(t, epc, generateIdenticalVersions(clusterSize, currentVersionStr)) + e2e.DowngradeCancel(t, epc, generateIdenticalVersions(clusterSize, currentVersion)) return // No need to perform downgrading, end the test here } e2e.DowngradeEnable(t, epc, lastVersion) if triggerCancellation == cancelRightAfterEnable { t.Logf("Cancelling downgrade right after enabling (no node is downgraded yet)") - e2e.DowngradeCancel(t, epc, generateIdenticalVersions(clusterSize, currentVersionStr)) + e2e.DowngradeCancel(t, epc, generateIdenticalVersions(clusterSize, currentVersion)) return // No need to perform downgrading, end the test here } membersToChange := rand.Perm(len(epc.Procs))[:cancellationSize] - t.Logf(fmt.Sprintf("Elect members for operations"), zap.Any("members", membersToChange)) + t.Logf(fmt.Sprintln("Elect members for operations"), zap.Any("members", membersToChange)) t.Logf("Starting downgrade process to %q", lastVersionStr) err = e2e.DowngradeUpgradeMembersByID(t, nil, epc, membersToChange, currentVersion, lastClusterVersion) require.NoError(t, err) - e2e.AssertProcessLogs(t, leader(t, epc), "the cluster has been downgraded") + if len(membersToChange) == len(epc.Procs) { + e2e.AssertProcessLogs(t, leader(t, epc), "the cluster has been downgraded") + } t.Log("Downgrade complete") afterMembers, afterKV := getMembersAndKeys(t, cc) @@ -171,6 +173,11 @@ func testDowngradeUpgrade(t *testing.T, cancellationSize int, clusterSize int, t t.Log("Waiting health interval to required to make membership changes") time.Sleep(etcdserver.HealthInterval) } + + if triggerCancellation == cancelAfterDowngrading { + e2e.DowngradeCancel(t, epc, generatePartialCancellationVersions(clusterSize, membersToChange, lastClusterVersion)) + } + t.Log("Adding learner to test membership, but avoid breaking quorum") resp, err = cc.MemberAddAsLearner(context.Background(), "fake2", []string{"http://127.0.0.1:1002"}) require.NoError(t, err) @@ -285,16 +292,34 @@ func getMembersAndKeys(t *testing.T, cc *e2e.EtcdctlV3) (*clientv3.MemberListRes return members, kvs } -func generateIdenticalVersions(clusterSize int, currentVersion string) []*version.Versions { +func generateIdenticalVersions(clusterSize int, ver *semver.Version) []*version.Versions { + ret := make([]*version.Versions, clusterSize) + + for i := range clusterSize { + ret[i] = &version.Versions{ + Cluster: ver.String(), + Server: ver.String(), + Storage: ver.String(), + } + } + + return ret +} + +func generatePartialCancellationVersions(clusterSize int, membersToChange []int, ver *semver.Version) []*version.Versions { ret := make([]*version.Versions, clusterSize) for i := range clusterSize { ret[i] = &version.Versions{ - Cluster: currentVersion, - Server: currentVersion, - Storage: currentVersion, + Cluster: ver.String(), + Server: e2e.OffsetMinor(ver, 1).String(), + Storage: "", } } + for i := range membersToChange { + ret[membersToChange[i]].Server = ver.String() + } + return ret } diff --git a/tests/framework/e2e/downgrade.go b/tests/framework/e2e/downgrade.go index 129a1d49498..2d6d4c32796 100644 --- a/tests/framework/e2e/downgrade.go +++ b/tests/framework/e2e/downgrade.go @@ -44,7 +44,7 @@ func DowngradeEnable(t *testing.T, epc *EtcdProcessCluster, ver *semver.Version) for i := 0; i < len(epc.Procs); i++ { ValidateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{ Cluster: ver.String(), - Server: offsetMinor(ver, 1).String(), + Server: OffsetMinor(ver, 1).String(), Storage: ver.String(), }) AssertProcessLogs(t, epc.Procs[i], "The server is ready to downgrade") @@ -75,7 +75,7 @@ func DowngradeCancel(t *testing.T, epc *EtcdProcessCluster, versions []*version. func DowngradeUpgradeMembers(t *testing.T, lg *zap.Logger, clus *EtcdProcessCluster, numberOfMembersToChange int, currentVersion, targetVersion *semver.Version) error { membersToChange := rand.Perm(len(clus.Procs))[:numberOfMembersToChange] - t.Logf(fmt.Sprintf("Elect members for operations"), zap.Any("members", membersToChange)) + t.Logf(fmt.Sprintln("Elect members for operations"), zap.Any("members", membersToChange)) return DowngradeUpgradeMembersByID(t, lg, clus, membersToChange, currentVersion, targetVersion) } @@ -150,8 +150,8 @@ func ValidateVersion(t *testing.T, cfg *EtcdProcessClusterConfig, member EtcdPro }) } -// offsetMinor returns the version with offset from the original minor, with the same major. -func offsetMinor(v *semver.Version, offset int) *semver.Version { +// OffsetMinor returns the version with offset from the original minor, with the same major. +func OffsetMinor(v *semver.Version, offset int) *semver.Version { var minor int64 if offset >= 0 { minor = v.Minor + int64(offset)