test(e2e): delete MantleBackups after backup-failure tests to avoid interfering with subsequent tests

The test "should handle removal of MantleBackup in secondary k8s cluster
during an incremental backup" was failing with a timeout.

When the previous test (primary MantleBackup deletion scenario) finishes, a
residual secondary MantleBackup is left behind in "waiting for export
data" state because its primary was deleted before the upload completed. When
the next test pauses RGW, the secondary reconciler gets blocked on S3
operations triggered by processing this residual MantleBackup, causing
`kubectl delete --timeout=3m` issued against the current test's secondary
MantleBackup to expire before finalizeSecondary could run.

Add CleanupMantleBackups to delete all MantleBackups on both clusters at the
end of each backup-failure test via DeferCleanup, so that no residual
MantleBackups remain when the next test starts.

Signed-off-by: Kohya Shiozaki <kouyan120706@gmail.com>

Author: skoya76

test/e2e/multik8s/backup_failure_test.go

@@ -46,6 +46,7 @@ var _ = Describe("backup failure", Label("backup-failure"), func() {
 		WaitTemporaryJobsDeleted(ctx, primaryMB, secondaryMB)
 		WaitTemporaryPVCsDeleted(ctx, primaryMB, secondaryMB)
 		WaitTemporarySecondaryPVsDeleted(ctx, secondaryMB)
+		CleanupMantleBackups(namespace)
 	})
 
 	It("should handle removal of MantleBackup in secondary k8s cluster during a full backup", func(ctx SpecContext) {
@@ -95,6 +96,7 @@ var _ = Describe("backup failure", Label("backup-failure"), func() {
 		Expect(err).NotTo(HaveOccurred())
 		WaitTemporaryResourcesDeleted(ctx, primaryMB, secondaryMB0)
 		WaitTemporaryResourcesDeleted(ctx, primaryMB, secondaryMB1)
+		CleanupMantleBackups(namespace)
 	})
 
 	It("should handle removal of MantleBackup in primary k8s cluster during an incremental backup",
@@ -146,6 +148,7 @@ var _ = Describe("backup failure", Label("backup-failure"), func() {
 			WaitTemporaryJobsDeleted(ctx, primaryMB, secondaryMB)
 			WaitTemporaryPVCsDeleted(ctx, primaryMB, secondaryMB)
 			WaitTemporarySecondaryPVsDeleted(ctx, secondaryMB)
+			CleanupMantleBackups(namespace)
 		})
 
 	It("should handle removal of MantleBackup in secondary k8s cluster during an incremental backup",
@@ -206,5 +209,6 @@ var _ = Describe("backup failure", Label("backup-failure"), func() {
 			Expect(err).NotTo(HaveOccurred())
 			WaitTemporaryResourcesDeleted(ctx, primaryMB1, secondaryMB10)
 			WaitTemporaryResourcesDeleted(ctx, primaryMB1, secondaryMB11)
+			CleanupMantleBackups(namespace)
 		})
 })

test/e2e/multik8s/testutil/util.go

@@ -486,6 +486,16 @@ func SetupEnvironment(namespace string) {
 	}).Should(Succeed())
 }
 
+func CleanupMantleBackups(namespace string) {
+	GinkgoHelper()
+	_, _, err := Kubectl(PrimaryK8sCluster, nil,
+		"delete", "mantlebackup", "--all", "-n", namespace, "--timeout=5m", "--ignore-not-found")
+	Expect(err).NotTo(HaveOccurred())
+	_, _, err = Kubectl(SecondaryK8sCluster, nil,
+		"delete", "mantlebackup", "--all", "-n", namespace, "--timeout=5m", "--ignore-not-found")
+	Expect(err).NotTo(HaveOccurred())
+}
+
 func CreatePod(cluster int, namespace, podName, pvcName string) {
 	GinkgoHelper()
 	err := applyPodMountVolumeTemplate(cluster, namespace, podName, pvcName)