Skip to content

Commit 42ba24a

Browse files
ushitora-anqouushitora-anqou
committed
test: e2e: add additional tests for verification
Signed-off-by: Ryotaro Banno <ryotaro.banno@gmail.com>
1 parent 65f04d9 commit 42ba24a

7 files changed

Lines changed: 202 additions & 25 deletions

File tree

‎internal/controller/mantlebackup_controller.go‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ const (
6969
MantleExportDataPVCPrefix = "mantle-export-"
7070
MantleImportJobPrefix = "mantle-import-"
7171
mantleVerifyImagePrefix = "mantle-verify-"
72-
mantleVerifyJobPrefix = "mantle-verify-"
72+
MantleVerifyJobPrefix = "mantle-verify-"
7373
mantleVerifyPVCPrefix = "mantle-verify-"
7474
mantleVerifyPVPrefix = "mantle-verify-"
7575
MantleZeroOutJobPrefix = "mantle-zeroout-"
@@ -717,7 +717,7 @@ func (r *MantleBackupReconciler) verify(
717717
corev1.ResourceList{
718718
corev1.ResourceStorage: *resource.NewQuantity(*backup.Status.SnapSize, resource.BinarySI),
719719
},
720-
makeVerifyPVName(backup),
720+
MakeVerifyPVName(backup),
721721
labelComponentVerifyVolume,
722722
true,
723723
); err != nil {
@@ -727,8 +727,8 @@ func (r *MantleBackupReconciler) verify(
727727
// create a PVC with the PV
728728
if err := r.createOrUpdateStaticPVC(
729729
ctx,
730-
makeVerifyPVCName(backup),
731-
makeVerifyPVName(backup),
730+
MakeVerifyPVCName(backup),
731+
MakeVerifyPVName(backup),
732732
labelComponentVerifyVolume,
733733
storedPVC.Spec.Resources,
734734
); err != nil {
@@ -738,14 +738,14 @@ func (r *MantleBackupReconciler) verify(
738738
// create a Job to execute e2fsck on the PVC
739739
if err := r.createOrUpdateVerifyJob(
740740
ctx,
741-
makeVerifyJobName(backup),
742-
makeVerifyPVCName(backup),
741+
MakeVerifyJobName(backup),
742+
MakeVerifyPVCName(backup),
743743
); err != nil {
744744
return fmt.Errorf("failed to create a Job to execute e2fsck on the PVC: %w", err)
745745
}
746746

747747
// wait for the Job to complete
748-
jobFinished, jobSucceeded, err := r.checkJobStatus(ctx, makeVerifyJobName(backup))
748+
jobFinished, jobSucceeded, err := r.checkJobStatus(ctx, MakeVerifyJobName(backup))
749749
if err != nil {
750750
return fmt.Errorf("failed to check the Job status: %w", err)
751751
}
@@ -1541,15 +1541,15 @@ func MakeVerifyImageName(target *mantlev1.MantleBackup) string {
15411541
return mantleVerifyImagePrefix + string(target.GetUID())
15421542
}
15431543

1544-
func makeVerifyJobName(target *mantlev1.MantleBackup) string {
1545-
return mantleVerifyJobPrefix + string(target.GetUID())
1544+
func MakeVerifyJobName(target *mantlev1.MantleBackup) string {
1545+
return MantleVerifyJobPrefix + string(target.GetUID())
15461546
}
15471547

1548-
func makeVerifyPVCName(target *mantlev1.MantleBackup) string {
1548+
func MakeVerifyPVCName(target *mantlev1.MantleBackup) string {
15491549
return mantleVerifyPVCPrefix + string(target.GetUID())
15501550
}
15511551

1552-
func makeVerifyPVName(target *mantlev1.MantleBackup) string {
1552+
func MakeVerifyPVName(target *mantlev1.MantleBackup) string {
15531553
return mantleVerifyPVPrefix + string(target.GetUID())
15541554
}
15551555

@@ -2853,7 +2853,7 @@ func (r *MantleBackupReconciler) deleteAllExportDataPVCs(ctx context.Context, ba
28532853

28542854
func (r *MantleBackupReconciler) deleteVerifyJob(ctx context.Context, backup *mantlev1.MantleBackup) error {
28552855
var job batchv1.Job
2856-
job.SetName(makeVerifyJobName(backup))
2856+
job.SetName(MakeVerifyJobName(backup))
28572857
job.SetNamespace(r.managedCephClusterID)
28582858
if err := r.Delete(ctx, &job, &client.DeleteOptions{
28592859
PropagationPolicy: ptr.To(metav1.DeletePropagationBackground),
@@ -2865,7 +2865,7 @@ func (r *MantleBackupReconciler) deleteVerifyJob(ctx context.Context, backup *ma
28652865

28662866
func (r *MantleBackupReconciler) deleteVerifyPV(ctx context.Context, backup *mantlev1.MantleBackup) error {
28672867
var pv corev1.PersistentVolume
2868-
pv.SetName(makeVerifyPVName(backup))
2868+
pv.SetName(MakeVerifyPVName(backup))
28692869
if err := r.Delete(ctx, &pv); err != nil && !aerrors.IsNotFound(err) {
28702870
return fmt.Errorf("failed to delete verify PV: %w", err)
28712871
}
@@ -2874,7 +2874,7 @@ func (r *MantleBackupReconciler) deleteVerifyPV(ctx context.Context, backup *man
28742874

28752875
func (r *MantleBackupReconciler) deleteVerifyPVC(ctx context.Context, backup *mantlev1.MantleBackup) error {
28762876
var pvc corev1.PersistentVolumeClaim
2877-
pvc.SetName(makeVerifyPVCName(backup))
2877+
pvc.SetName(MakeVerifyPVCName(backup))
28782878
pvc.SetNamespace(r.managedCephClusterID)
28792879
if err := r.Delete(ctx, &pvc); err != nil && !aerrors.IsNotFound(err) {
28802880
return fmt.Errorf("failed to delete verify PVC: %w", err)

‎internal/controller/mantlebackup_controller_test.go‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2562,7 +2562,7 @@ var _ = Describe("MantleBackupReconciler", func() {
25622562
GinkgoHelper()
25632563
var pv corev1.PersistentVolume
25642564
err := k8sClient.Get(ctx, types.NamespacedName{
2565-
Name: makeVerifyPVName(backup),
2565+
Name: MakeVerifyPVName(backup),
25662566
}, &pv)
25672567
Expect(err).NotTo(HaveOccurred())
25682568

@@ -2596,7 +2596,7 @@ var _ = Describe("MantleBackupReconciler", func() {
25962596
GinkgoHelper()
25972597
var pvc corev1.PersistentVolumeClaim
25982598
err := k8sClient.Get(ctx, types.NamespacedName{
2599-
Name: makeVerifyPVCName(backup),
2599+
Name: MakeVerifyPVCName(backup),
26002600
Namespace: podNamespace,
26012601
}, &pvc)
26022602
Expect(err).NotTo(HaveOccurred())
@@ -2608,15 +2608,15 @@ var _ = Describe("MantleBackupReconciler", func() {
26082608
Resources: pvcSrc.Spec.Resources,
26092609
StorageClassName: ptr.To(""),
26102610
VolumeMode: ptr.To(corev1.PersistentVolumeBlock),
2611-
VolumeName: makeVerifyPVName(backup),
2611+
VolumeName: MakeVerifyPVName(backup),
26122612
}))
26132613
}
26142614

26152615
checkJobSpec := func(ctx context.Context, backup *mantlev1.MantleBackup) *batchv1.Job {
26162616
GinkgoHelper()
26172617
var job batchv1.Job
26182618
err := k8sClient.Get(ctx, types.NamespacedName{
2619-
Name: makeVerifyJobName(backup),
2619+
Name: MakeVerifyJobName(backup),
26202620
Namespace: podNamespace,
26212621
}, &job)
26222622
Expect(err).NotTo(HaveOccurred())
@@ -2652,7 +2652,7 @@ var _ = Describe("MantleBackupReconciler", func() {
26522652
Name: "verify-rbd",
26532653
VolumeSource: corev1.VolumeSource{
26542654
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
2655-
ClaimName: makeVerifyPVCName(backup),
2655+
ClaimName: MakeVerifyPVCName(backup),
26562656
},
26572657
},
26582658
}))
@@ -2740,9 +2740,9 @@ var _ = Describe("MantleBackupReconciler", func() {
27402740
It("generates correct names", func(ctx SpecContext) {
27412741
for _, backup := range []*mantlev1.MantleBackup{backupSuccess, backupFail} {
27422742
Expect(MakeVerifyImageName(backup)).To(Equal(mantleVerifyImagePrefix + string(backup.GetUID())))
2743-
Expect(makeVerifyJobName(backup)).To(Equal(mantleVerifyJobPrefix + string(backup.GetUID())))
2744-
Expect(makeVerifyPVName(backup)).To(Equal(mantleVerifyPVPrefix + string(backup.GetUID())))
2745-
Expect(makeVerifyPVCName(backup)).To(Equal(mantleVerifyPVCPrefix + string(backup.GetUID())))
2743+
Expect(MakeVerifyJobName(backup)).To(Equal(MantleVerifyJobPrefix + string(backup.GetUID())))
2744+
Expect(MakeVerifyPVName(backup)).To(Equal(mantleVerifyPVPrefix + string(backup.GetUID())))
2745+
Expect(MakeVerifyPVCName(backup)).To(Equal(mantleVerifyPVCPrefix + string(backup.GetUID())))
27462746
}
27472747
})
27482748

@@ -2764,7 +2764,7 @@ var _ = Describe("MantleBackupReconciler", func() {
27642764
// Update the Job to Completed
27652765
var currentJob batchv1.Job
27662766
err := k8sClient.Get(ctx, types.NamespacedName{
2767-
Name: makeVerifyJobName(backupSuccess),
2767+
Name: MakeVerifyJobName(backupSuccess),
27682768
Namespace: podNamespace,
27692769
}, &currentJob)
27702770
Expect(err).NotTo(HaveOccurred())
@@ -2812,7 +2812,7 @@ var _ = Describe("MantleBackupReconciler", func() {
28122812
// Update the Job to Failed
28132813
var currentJob batchv1.Job
28142814
err = k8sClient.Get(ctx, types.NamespacedName{
2815-
Name: makeVerifyJobName(backupFail),
2815+
Name: MakeVerifyJobName(backupFail),
28162816
Namespace: podNamespace,
28172817
}, &currentJob)
28182818
Expect(err).NotTo(HaveOccurred())

‎test/e2e/multik8s/full_backup_test.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,11 @@ var _ = Describe("full backup", Label("full-backup"), func() {
215215
Expect(err).NotTo(HaveOccurred())
216216
WaitTemporaryResourcesDeleted(ctx, primaryMB1, secondaryMB1)
217217

218+
// Make sure verification step has completed in both clusters.
219+
WaitMantleBackupVerified(PrimaryK8sCluster, namespace, backupName0)
220+
WaitMantleBackupVerified(PrimaryK8sCluster, namespace, backupName1)
221+
WaitMantleBackupVerified(SecondaryK8sCluster, namespace, backupName1)
222+
218223
// Make sure M1 and M1' have the same contents.
219224
EnsureCorrectRestoration(PrimaryK8sCluster, ctx, namespace, backupName1, restoreName1, writtenDataHash1)
220225
EnsureCorrectRestoration(SecondaryK8sCluster, ctx, namespace, backupName1, restoreName1, writtenDataHash1)
@@ -255,6 +260,11 @@ var _ = Describe("full backup", Label("full-backup"), func() {
255260
Expect(err).NotTo(HaveOccurred())
256261
WaitTemporaryResourcesDeleted(ctx, primaryMB1, secondaryMB1)
257262

263+
// Make sure verification step has completed in both clusters.
264+
WaitMantleBackupVerified(SecondaryK8sCluster, namespace, backupName0)
265+
WaitMantleBackupVerified(PrimaryK8sCluster, namespace, backupName1)
266+
WaitMantleBackupVerified(SecondaryK8sCluster, namespace, backupName1)
267+
258268
// Make sure M1 and M1' have the same contents.
259269
EnsureCorrectRestoration(PrimaryK8sCluster, ctx, namespace, backupName1, restoreName1, writtenDataHash1)
260270
EnsureCorrectRestoration(SecondaryK8sCluster, ctx, namespace, backupName1, restoreName1, writtenDataHash1)

‎test/e2e/multik8s/incr_backup_test.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,12 @@ var _ = Describe("incremental backup", Label("incr-backup"), func() {
4444
Expect(err).NotTo(HaveOccurred())
4545
WaitTemporaryResourcesDeleted(ctx, primaryMB1, secondaryMB1)
4646

47+
// Make sure verification step has completed in both clusters.
48+
WaitMantleBackupVerified(PrimaryK8sCluster, namespace, backupName0)
49+
WaitMantleBackupVerified(SecondaryK8sCluster, namespace, backupName0)
50+
WaitMantleBackupVerified(PrimaryK8sCluster, namespace, backupName1)
51+
WaitMantleBackupVerified(SecondaryK8sCluster, namespace, backupName1)
52+
4753
// Make sure M1 and M1' have the same contents.
4854
EnsureCorrectRestoration(PrimaryK8sCluster, ctx, namespace, backupName1, restoreName1, writtenDataHash1)
4955
EnsureCorrectRestoration(SecondaryK8sCluster, ctx, namespace, backupName1, restoreName1, writtenDataHash1)
@@ -91,6 +97,13 @@ var _ = Describe("incremental backup", Label("incr-backup"), func() {
9197
Expect(err).NotTo(HaveOccurred())
9298
WaitTemporaryResourcesDeleted(ctx, primaryMB2, secondaryMB2)
9399

100+
// Make sure verification step has completed in both clusters.
101+
WaitMantleBackupVerified(PrimaryK8sCluster, namespace, backupName0)
102+
WaitMantleBackupVerified(SecondaryK8sCluster, namespace, backupName0)
103+
WaitMantleBackupVerified(PrimaryK8sCluster, namespace, backupName1)
104+
WaitMantleBackupVerified(PrimaryK8sCluster, namespace, backupName2)
105+
WaitMantleBackupVerified(SecondaryK8sCluster, namespace, backupName2)
106+
94107
// Make sure M2 and M2' have the same contents.
95108
EnsureCorrectRestoration(PrimaryK8sCluster, ctx, namespace, backupName2, restoreName2, writtenDataHash2)
96109
EnsureCorrectRestoration(SecondaryK8sCluster, ctx, namespace, backupName2, restoreName2, writtenDataHash2)
@@ -148,6 +161,13 @@ var _ = Describe("incremental backup", Label("incr-backup"), func() {
148161
Expect(err).NotTo(HaveOccurred())
149162
WaitTemporaryResourcesDeleted(ctx, primaryMB2, secondaryMB2)
150163

164+
// Make sure verification step has completed in both clusters.
165+
WaitMantleBackupVerified(PrimaryK8sCluster, namespace, backupName0)
166+
WaitMantleBackupVerified(SecondaryK8sCluster, namespace, backupName0)
167+
WaitMantleBackupVerified(SecondaryK8sCluster, namespace, backupName1)
168+
WaitMantleBackupVerified(PrimaryK8sCluster, namespace, backupName2)
169+
WaitMantleBackupVerified(SecondaryK8sCluster, namespace, backupName2)
170+
151171
// Make sure M2 and M2' have the same contents.
152172
EnsureCorrectRestoration(PrimaryK8sCluster, ctx, namespace, backupName2, restoreName2, writtenDataHash2)
153173
EnsureCorrectRestoration(SecondaryK8sCluster, ctx, namespace, backupName2, restoreName2, writtenDataHash2)

‎test/e2e/multik8s/testutil/util.go‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -811,12 +811,26 @@ func WaitComponentJobsDeleted(
811811
}).Should(Succeed())
812812
}
813813

814+
func WaitJobDeleted(ctx SpecContext, cluster int, namespace, jobName string) {
815+
GinkgoHelper()
816+
By("waiting for a PVC to be deleted")
817+
Eventually(ctx, func(g Gomega) {
818+
jobs, err := GetJobList(cluster, CephClusterNamespace)
819+
g.Expect(err).NotTo(HaveOccurred())
820+
exist := slices.ContainsFunc(jobs.Items, func(job batchv1.Job) bool {
821+
return job.GetName() == jobName
822+
})
823+
g.Expect(exist).To(BeFalse())
824+
}).Should(Succeed())
825+
}
826+
814827
func WaitTemporaryPrimaryJobsDeleted(ctx SpecContext, primaryMB *mantlev1.MantleBackup) {
815828
GinkgoHelper()
816829
WaitComponentJobsDeleted(ctx, PrimaryK8sCluster, CephClusterNamespace,
817830
controller.MantleExportJobPrefix, primaryMB)
818831
WaitComponentJobsDeleted(ctx, PrimaryK8sCluster, CephClusterNamespace,
819832
controller.MantleUploadJobPrefix, primaryMB)
833+
WaitJobDeleted(ctx, PrimaryK8sCluster, CephClusterNamespace, controller.MakeVerifyJobName(primaryMB))
820834
}
821835

822836
func WaitTemporarySecondaryJobsDeleted(ctx SpecContext, secondaryMB *mantlev1.MantleBackup) {
@@ -825,6 +839,7 @@ func WaitTemporarySecondaryJobsDeleted(ctx SpecContext, secondaryMB *mantlev1.Ma
825839
controller.MantleImportJobPrefix, secondaryMB)
826840
WaitComponentJobsDeleted(ctx, SecondaryK8sCluster, CephClusterNamespace,
827841
controller.MantleZeroOutJobPrefix, secondaryMB)
842+
WaitJobDeleted(ctx, SecondaryK8sCluster, CephClusterNamespace, controller.MakeVerifyJobName(secondaryMB))
828843
}
829844

830845
func WaitTemporaryJobsDeleted(ctx SpecContext, primaryMB, secondaryMB *mantlev1.MantleBackup) {
@@ -863,11 +878,13 @@ func WaitTemporaryPrimaryPVCsDeleted(ctx SpecContext, primaryMB *mantlev1.Mantle
863878
GinkgoHelper()
864879
WaitPVCsDeleted(ctx, PrimaryK8sCluster, CephClusterNamespace,
865880
fmt.Sprintf("%s%s-", controller.MantleExportDataPVCPrefix, string(primaryMB.GetUID())))
881+
WaitPVCDeleted(ctx, PrimaryK8sCluster, CephClusterNamespace, controller.MakeVerifyPVCName(primaryMB))
866882
}
867883

868884
func WaitTemporarySecondaryPVCsDeleted(ctx SpecContext, secondaryMB *mantlev1.MantleBackup) {
869885
GinkgoHelper()
870886
WaitPVCDeleted(ctx, SecondaryK8sCluster, CephClusterNamespace, controller.MakeZeroOutPVCName(secondaryMB))
887+
WaitPVCDeleted(ctx, SecondaryK8sCluster, CephClusterNamespace, controller.MakeVerifyPVCName(secondaryMB))
871888
}
872889

873890
func WaitTemporaryPVCsDeleted(ctx SpecContext, primaryMB, secondaryMB *mantlev1.MantleBackup) {
@@ -889,9 +906,21 @@ func WaitPVDeleted(ctx SpecContext, cluster int, pvName string) {
889906
}).Should(Succeed())
890907
}
891908

909+
func WaitTemporaryPrimaryPVsDeleted(ctx SpecContext, primaryMB *mantlev1.MantleBackup) {
910+
GinkgoHelper()
911+
WaitPVDeleted(ctx, PrimaryK8sCluster, controller.MakeVerifyPVName(primaryMB))
912+
}
913+
892914
func WaitTemporarySecondaryPVsDeleted(ctx SpecContext, secondaryMB *mantlev1.MantleBackup) {
893915
GinkgoHelper()
894916
WaitPVDeleted(ctx, SecondaryK8sCluster, controller.MakeZeroOutPVName(secondaryMB))
917+
WaitPVDeleted(ctx, SecondaryK8sCluster, controller.MakeVerifyPVName(secondaryMB))
918+
}
919+
920+
func WaitTemporaryPVsDeleted(ctx SpecContext, primaryMB, secondaryMB *mantlev1.MantleBackup) {
921+
GinkgoHelper()
922+
WaitTemporaryPrimaryPVsDeleted(ctx, primaryMB)
923+
WaitTemporarySecondaryPVsDeleted(ctx, secondaryMB)
895924
}
896925

897926
func WaitTemporaryS3ObjectsDeleted(ctx SpecContext, primaryMB *mantlev1.MantleBackup) {
@@ -925,7 +954,7 @@ func WaitTemporaryResourcesDeleted(ctx SpecContext, primaryMB, secondaryMB *mant
925954
GinkgoHelper()
926955
WaitTemporaryJobsDeleted(ctx, primaryMB, secondaryMB)
927956
WaitTemporaryPVCsDeleted(ctx, primaryMB, secondaryMB)
928-
WaitTemporarySecondaryPVsDeleted(ctx, secondaryMB)
957+
WaitTemporaryPVsDeleted(ctx, primaryMB, secondaryMB)
929958
WaitVerifyRBDImageDeleted(ctx, PrimaryK8sCluster, primaryMB)
930959
WaitVerifyRBDImageDeleted(ctx, SecondaryK8sCluster, secondaryMB)
931960
WaitTemporaryS3ObjectsDeleted(ctx, primaryMB)

‎test/e2e/singlek8s/suite_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ func TestMtest(t *testing.T) {
3232
var _ = Describe("Mantle", Ordered, func() {
3333
Context("backup", backupTestSuite)
3434
Context("restore", restoreTestSuite)
35+
Context("verification", verifyTestSuite)
3536
Context("multi Rook/Ceph env", multiRookCephTestSuite)
3637

3738
BeforeAll(func() {

0 commit comments

Comments
 (0)