test: singlek8s: improve test for multi-controller env

Signed-off-by: Yuji Ito <llamerada.jp@gmail.com>

Author: llamerada-jp

test/e2e/singlek8s/multi_rook_ceph_test.go

@@ -2,10 +2,14 @@ package singlek8s
 
 import (
 	"fmt"
+	"strings"
 
+	mantlev1 "github.com/cybozu-go/mantle/api/v1"
+	"github.com/cybozu-go/mantle/internal/controller"
 	"github.com/cybozu-go/mantle/test/util"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
 )
 
 type multiRookCephTest struct {
@@ -96,9 +100,26 @@ func (test *multiRookCephTest) testMain() {
 	var srcImageName1, srcImageName2, srcImageNameDummy string
 	var expectImageAndSnaps1, expectImageAndSnaps2 []string
 	var dummySnapInfo, dummyCloneInfo *rbdInfo
+	var backup1, backup2 *mantlev1.MantleBackup
 
-	It("create PVC, backup, and restore in the tenant namespace 1", func() {
-		test.createPVCBackupRestoreWithData(test.tenantNamespace1, test.storageClassName1, testData1)
+	// --- Cluster 1 backup lifecycle ---
+
+	It("create PVC and write data in tenant namespace 1", func() {
+		err := applyPVCTemplate(test.tenantNamespace1, test.pvcName, test.storageClassName1)
+		Expect(err).NotTo(HaveOccurred())
+		err = writeTestData(test.tenantNamespace1, test.pvcName, testData1)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("create backup in tenant namespace 1 and wait for snapshot", func() {
+		err := applyMantleBackupTemplate(test.tenantNamespace1, test.pvcName, test.mantleBackupName)
+		Expect(err).NotTo(HaveOccurred())
+
+		Eventually(func(g Gomega) {
+			ok, err := isMantleBackupSnapshotCaptured(test.tenantNamespace1, test.mantleBackupName)
+			g.Expect(err).NotTo(HaveOccurred())
+			g.Expect(ok).To(BeTrue())
+		}).Should(Succeed())
 
 		pvName, err := getPVFromPVC(test.tenantNamespace1, test.pvcName)
 		Expect(err).NotTo(HaveOccurred())
@@ -111,6 +132,86 @@ func (test *multiRookCephTest) testMain() {
 		}
 	})
 
+	It("check labels on MantleBackup after backup in tenant namespace 1", func() {
+		pvc, err := getObject[corev1.PersistentVolumeClaim]("pvc", test.tenantNamespace1, test.pvcName)
+		Expect(err).NotTo(HaveOccurred())
+
+		mb, err := getMB(test.tenantNamespace1, test.mantleBackupName)
+		Expect(err).NotTo(HaveOccurred())
+
+		mantleLabels := make(map[string]string)
+		for k, v := range mb.GetLabels() {
+			if strings.HasPrefix(k, "mantle.cybozu.io/") {
+				mantleLabels[k] = v
+			}
+		}
+		Expect(mantleLabels).To(Equal(map[string]string{
+			"mantle.cybozu.io/cluster-id":                  cephCluster1Namespace,
+			"mantle.cybozu.io/local-backup-target-pvc-uid": string(pvc.GetUID()),
+		}))
+	})
+
+	It("verify for tenant namespace 1 runs only in ceph cluster 1, is cleaned up, and never appears in ceph cluster 2", func() {
+		// The MantleBackup already exists, so the verify resource names (derived from its
+		// UID) are known now, before verify runs. This lets us start monitoring ceph cluster 2
+		// from the beginning of the verify lifecycle.
+		mb, err := getMB(test.tenantNamespace1, test.mantleBackupName)
+		Expect(err).NotTo(HaveOccurred())
+		backup1 = mb
+
+		jobName := controller.MakeVerifyJobName(backup1)
+		pvcName := controller.MakeVerifyPVCName(backup1)
+
+		// Continuously monitor until ceph cluster 1 finishes verify and cleans up its
+		// resources. On every poll, hard-assert (via the outer Expect, not g.Expect) that
+		// the verify resources never appear in ceph cluster 2, so even a transient
+		// appearance fails the test immediately instead of being tolerated as a retryable
+		// condition. A single point-in-time check would miss resources that are created
+		// briefly in the wrong namespace and then cleaned up quickly.
+		Eventually(func(g Gomega) {
+			// The verify resources must NEVER appear in ceph cluster 2 at any point, so it doesn't use 'g'.
+			existsJob2, err := checkJobExists(cephCluster2Namespace, jobName)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(existsJob2).To(BeFalse(), "verify job must not run in ceph cluster 2")
+			existsPVC2, err := checkPVCExists(cephCluster2Namespace, pvcName)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(existsPVC2).To(BeFalse(), "verify PVC must not be created in ceph cluster 2")
+
+			mb, err := getMB(test.tenantNamespace1, test.mantleBackupName)
+			g.Expect(err).NotTo(HaveOccurred())
+			// If the ceph cluster 2 controller mistakenly runs verify for this backup,
+			// the verify job fails because the RBD snapshot does not exist in ceph cluster 2,
+			// which causes IsVerifiedFalse to be set instead of IsVerifiedTrue.
+			Expect(mb.IsVerifiedFalse()).To(BeFalse(),
+				"verify failed: ceph cluster 2 controller may have run verify for ceph cluster 1's backup")
+
+			// Terminating condition: verify succeeded in ceph cluster 1 and its resources
+			// have been cleaned up.
+			g.Expect(mb.IsVerifiedTrue()).To(BeTrue())
+			existsJob1, err := checkJobExists(cephCluster1Namespace, jobName)
+			g.Expect(err).NotTo(HaveOccurred())
+			g.Expect(existsJob1).To(BeFalse(), "verify job should be cleaned up from ceph cluster 1")
+			existsPVC1, err := checkPVCExists(cephCluster1Namespace, pvcName)
+			g.Expect(err).NotTo(HaveOccurred())
+			g.Expect(existsPVC1).To(BeFalse(), "verify PVC should be cleaned up from ceph cluster 1")
+		}).Should(Succeed())
+	})
+
+	It("create restore in tenant namespace 1 and wait for ready", func() {
+		err := applyMantleRestoreTemplate(test.tenantNamespace1, test.mantleRestoreName, test.mantleBackupName)
+		Expect(err).NotTo(HaveOccurred())
+		Eventually(func() bool {
+			return isMantleRestoreReady(test.tenantNamespace1, test.mantleRestoreName)
+		}).Should(BeTrue())
+	})
+
+	// --- Dummy data ---
+	// Manually create RBD images/snaps/clones in ceph cluster 2 using the same names as
+	// those created by ceph cluster 1's controller. This lets the subsequent RBD state
+	// checks verify that ceph cluster 1's controller never touches ceph cluster 2's RBD
+	// images. No corresponding dummy data is placed in ceph cluster 1 because the test
+	// focuses on this one-directional isolation: cluster 1 must not interfere with cluster 2.
+
 	It("create the dummy RBD image & snap in the ceph cluster 2 with the same name as ceph cluster 1", func() {
 		By("get source image name from the PV/PVC")
 		pvName, err := getPVFromPVC(test.tenantNamespace1, test.pvcName)
@@ -136,7 +237,7 @@ func (test *multiRookCephTest) testMain() {
 		Expect(err).NotTo(HaveOccurred())
 	})
 
-	It("check the rbd images in the both clusters", func() {
+	It("check the rbd images in both clusters after cluster 1 backup", func() {
 		imageAndSnaps1, err := getImageAndSnapNames(cephCluster1Namespace, test.poolName)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(imageAndSnaps1).To(ConsistOf(expectImageAndSnaps1))
@@ -146,8 +247,24 @@ func (test *multiRookCephTest) testMain() {
 		Expect(imageAndSnaps2).To(ConsistOf(expectImageAndSnaps1))
 	})
 
-	It("create PVC, backup, and restore in the tenant namespace 2", func() {
-		test.createPVCBackupRestoreWithData(test.tenantNamespace2, test.storageClassName2, testData2)
+	// --- Cluster 2 backup lifecycle ---
+
+	It("create PVC and write data in tenant namespace 2", func() {
+		err := applyPVCTemplate(test.tenantNamespace2, test.pvcName, test.storageClassName2)
+		Expect(err).NotTo(HaveOccurred())
+		err = writeTestData(test.tenantNamespace2, test.pvcName, testData2)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("create backup in tenant namespace 2 and wait for snapshot", func() {
+		err := applyMantleBackupTemplate(test.tenantNamespace2, test.pvcName, test.mantleBackupName)
+		Expect(err).NotTo(HaveOccurred())
+
+		Eventually(func(g Gomega) {
+			ok, err := isMantleBackupSnapshotCaptured(test.tenantNamespace2, test.mantleBackupName)
+			g.Expect(err).NotTo(HaveOccurred())
+			g.Expect(ok).To(BeTrue())
+		}).Should(Succeed())
 
 		pvName, err := getPVFromPVC(test.tenantNamespace2, test.pvcName)
 		Expect(err).NotTo(HaveOccurred())
@@ -160,6 +277,73 @@ func (test *multiRookCephTest) testMain() {
 		}
 	})
 
+	It("check labels on MantleBackup after backup in tenant namespace 2", func() {
+		pvc, err := getObject[corev1.PersistentVolumeClaim]("pvc", test.tenantNamespace2, test.pvcName)
+		Expect(err).NotTo(HaveOccurred())
+
+		mb, err := getMB(test.tenantNamespace2, test.mantleBackupName)
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(mb.GetLabels()).To(HaveKeyWithValue("mantle.cybozu.io/cluster-id", cephCluster2Namespace))
+		Expect(mb.GetLabels()).To(HaveKeyWithValue("mantle.cybozu.io/local-backup-target-pvc-uid", string(pvc.GetUID())))
+	})
+
+	It("verify for tenant namespace 2 runs only in ceph cluster 2, is cleaned up, and never appears in ceph cluster 1", func() {
+		// The MantleBackup already exists, so the verify resource names (derived from its
+		// UID) are known now, before verify runs. This lets us start monitoring ceph cluster 1
+		// from the beginning of the verify lifecycle.
+		mb, err := getMB(test.tenantNamespace2, test.mantleBackupName)
+		Expect(err).NotTo(HaveOccurred())
+		backup2 = mb
+
+		jobName := controller.MakeVerifyJobName(backup2)
+		pvcName := controller.MakeVerifyPVCName(backup2)
+
+		// Continuously monitor until ceph cluster 2 finishes verify and cleans up its
+		// resources. On every poll, hard-assert (via the outer Expect, not g.Expect) that
+		// the verify resources never appear in ceph cluster 1, so even a transient
+		// appearance fails the test immediately instead of being tolerated as a retryable
+		// condition. A single point-in-time check would miss resources that are created
+		// briefly in the wrong namespace and then cleaned up quickly.
+		Eventually(func(g Gomega) {
+			// The verify resources must NEVER appear in ceph cluster 1 at any point, so it doesn't use 'g'.
+			existsJob1, err := checkJobExists(cephCluster1Namespace, jobName)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(existsJob1).To(BeFalse(), "verify job must not run in ceph cluster 1")
+			existsPVC1, err := checkPVCExists(cephCluster1Namespace, pvcName)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(existsPVC1).To(BeFalse(), "verify PVC must not be created in ceph cluster 1")
+
+			mb, err := getMB(test.tenantNamespace2, test.mantleBackupName)
+			g.Expect(err).NotTo(HaveOccurred())
+			// If the ceph cluster 1 controller mistakenly runs verify for this backup,
+			// the verify job fails because the RBD snapshot does not exist in ceph cluster 1,
+			// which causes IsVerifiedFalse to be set instead of IsVerifiedTrue.
+			Expect(mb.IsVerifiedFalse()).To(BeFalse(),
+				"verify failed: ceph cluster 1 controller may have run verify for ceph cluster 2's backup")
+
+			// Terminating condition: verify succeeded in ceph cluster 2 and its resources
+			// have been cleaned up.
+			g.Expect(mb.IsVerifiedTrue()).To(BeTrue())
+			existsJob2, err := checkJobExists(cephCluster2Namespace, jobName)
+			g.Expect(err).NotTo(HaveOccurred())
+			g.Expect(existsJob2).To(BeFalse(), "verify job should be cleaned up from ceph cluster 2")
+			existsPVC2, err := checkPVCExists(cephCluster2Namespace, pvcName)
+			g.Expect(err).NotTo(HaveOccurred())
+			g.Expect(existsPVC2).To(BeFalse(), "verify PVC should be cleaned up from ceph cluster 2")
+		}).Should(Succeed())
+	})
+
+	It("create restore in tenant namespace 2 and wait for ready", func() {
+		err := applyMantleRestoreTemplate(test.tenantNamespace2, test.mantleRestoreName, test.mantleBackupName)
+		Expect(err).NotTo(HaveOccurred())
+		Eventually(func() bool {
+			return isMantleRestoreReady(test.tenantNamespace2, test.mantleRestoreName)
+		}).Should(BeTrue())
+	})
+
+	// --- Cross-cluster RBD state and data integrity checks ---
+
 	It("check the rbd images in the both clusters", func() {
 		imageAndSnaps1, err := getImageAndSnapNames(cephCluster1Namespace, test.poolName)
 		Expect(err).NotTo(HaveOccurred())
@@ -176,6 +360,8 @@ func (test *multiRookCephTest) testMain() {
 		Expect(backupData2).To(Equal(testData2))
 	})
 
+	// --- Deletion isolation checks ---
+
 	It("delete the resources in the tenant namespace 1", func() {
 		test.deleteBackupRestore(test.tenantNamespace1)
 	})
@@ -230,23 +416,6 @@ func (test *multiRookCephTest) testMain() {
 	})
 }
 
-func (test *multiRookCephTest) createPVCBackupRestoreWithData(ns, sc string, data []byte) {
-	err := applyPVCTemplate(ns, test.pvcName, sc)
-	Expect(err).NotTo(HaveOccurred())
-	err = writeTestData(ns, test.pvcName, data)
-	Expect(err).NotTo(HaveOccurred())
-
-	err = applyMantleBackupTemplate(ns, test.pvcName, test.mantleBackupName)
-	Expect(err).NotTo(HaveOccurred())
-
-	err = applyMantleRestoreTemplate(ns, test.mantleRestoreName, test.mantleBackupName)
-	Expect(err).NotTo(HaveOccurred())
-
-	Eventually(func() bool {
-		return isMantleRestoreReady(ns, test.mantleRestoreName)
-	}).Should(BeTrue())
-}
-
 func (test *multiRookCephTest) deleteBackupRestore(ns string) {
 	_, stderr, err := kubectl("delete", "mantlerestore", "-n", ns, test.mantleRestoreName)
 	Expect(err).NotTo(HaveOccurred(), string(stderr))

test/e2e/singlek8s/util.go

@@ -538,3 +538,18 @@ func checkJobExists(namespace, jobName string) (bool, error) {
 
 	return found, nil
 }
+
+func checkPVCExists(namespace, pvcName string) (bool, error) {
+	stdout, _, err := kubectl("get", "pvc", "-n", namespace, "-o", "json")
+	if err != nil {
+		return false, err
+	}
+	var pvcList corev1.PersistentVolumeClaimList
+	if err := json.Unmarshal(stdout, &pvcList); err != nil {
+		return false, err
+	}
+
+	return slices.ContainsFunc(pvcList.Items, func(pvc corev1.PersistentVolumeClaim) bool {
+		return pvc.Name == pvcName
+	}), nil
+}