use verify Job

Author: ushitora-anqou

api/v1/mantlebackup_types.go

@@ -101,6 +101,10 @@ func (m *MantleBackup) IsSynced() bool {
 	return meta.IsStatusConditionTrue(m.Status.Conditions, BackupConditionSyncedToRemote)
 }
 
+func (m *MantleBackup) IsVerified() bool {
+	return meta.IsStatusConditionTrue(m.Status.Conditions, BackupConditionVerified)
+}
+
 func init() {
 	SchemeBuilder.Register(&MantleBackup{}, &MantleBackupList{})
 }

internal/controller/mantlebackup_controller.go

@@ -410,8 +410,10 @@ func (r *MantleBackupReconciler) reconcilePre(ctx context.Context, backup *mantl
 		return ctrl.Result{}, err
 	}
 
-	if err := r.provisionRBDSnapshot(ctx, backup, target); err != nil {
-		return ctrl.Result{}, err
+	if !backup.IsReady() {
+		if err := r.provisionRBDSnapshot(ctx, backup, target); err != nil {
+			return ctrl.Result{}, err
+		}
 	}
 
 	return ctrl.Result{}, nil
@@ -423,6 +425,13 @@ func (r *MantleBackupReconciler) reconcileAsStandalone(ctx context.Context, back
 		return result, err
 	}
 
+	if !backup.IsVerified() {
+		if err := r.verify(ctx, backup); err != nil {
+			return ctrl.Result{}, err
+		}
+		return requeueReconciliation(), nil
+	}
+
 	return r.primaryCleanup(ctx, backup)
 }
 
@@ -432,8 +441,21 @@ func (r *MantleBackupReconciler) reconcileAsPrimary(ctx context.Context, backup
 		return result, err
 	}
 
+	if !backup.IsVerified() {
+		if err := r.verify(ctx, backup); err != nil {
+			return ctrl.Result{}, err
+		}
+	}
+
 	if !backup.IsSynced() {
-		return r.replicate(ctx, backup)
+		result, err := r.replicate(ctx, backup)
+		if err != nil || !result.IsZero() {
+			return result, err
+		}
+	}
+
+	if !backup.IsVerified() || !backup.IsSynced() {
+		return requeueReconciliation(), nil
 	}
 
 	return r.primaryCleanup(ctx, backup)
@@ -477,7 +499,17 @@ func (r *MantleBackupReconciler) reconcileAsSecondary(ctx context.Context, backu
 	}
 
 	if !backup.IsReady() {
-		return r.startImport(ctx, backup, target)
+		result, err := r.startImport(ctx, backup, target)
+		if err != nil || !result.IsZero() {
+			return result, err
+		}
+	}
+
+	if backup.IsReady() && !backup.IsVerified() {
+		if err := r.verify(ctx, backup); err != nil {
+			return ctrl.Result{}, err
+		}
+		return requeueReconciliation(), nil
 	}
 
 	return r.secondaryCleanup(ctx, backup, true)
@@ -658,7 +690,6 @@ func (r *MantleBackupReconciler) replicateManifests(
 	return ctrl.Result{}, nil
 }
 
-//nolint:unused // TODO: Delete this line after starting use.
 func (r *MantleBackupReconciler) verify(
 	ctx context.Context,
 	backup *mantlev1.MantleBackup,
@@ -674,15 +705,15 @@ func (r *MantleBackupReconciler) verify(
 	}
 
 	// create a clone by the snapshot bound to the MB
-	if err := createCloneByPV(ctx, r.ceph, &storedPV, backup.Name, makeVerifyImageName(backup)); err != nil {
+	if err := createCloneByPV(ctx, r.ceph, &storedPV, backup.Name, MakeVerifyImageName(backup)); err != nil {
 		return fmt.Errorf("failed to create a clone by the snapshot: %w", err)
 	}
 
 	// create a static PV with the clone
 	if err := r.createOrUpdateStaticPV(
 		ctx,
 		&storedPV,
-		makeVerifyImageName(backup),
+		MakeVerifyImageName(backup),
 		corev1.ResourceList{
 			corev1.ResourceStorage: *resource.NewQuantity(*backup.Status.SnapSize, resource.BinarySI),
 		},
@@ -744,8 +775,6 @@ func (r *MantleBackupReconciler) verify(
 
 // checkJobStatus checks the status of the Job with the given name.
 // It returns (is job finished, is job succeeded, error).
-//
-//nolint:unused // TODO: Delete this line after starting use `verify`.
 func (r *MantleBackupReconciler) checkJobStatus(ctx context.Context, jobName string) (bool, bool, error) {
 	var job batchv1.Job
 	if err := r.Get(
@@ -799,11 +828,6 @@ func (r *MantleBackupReconciler) provisionRBDSnapshot(
 		return err
 	}
 
-	// If the given MantleBackup is not ready to use, create a new RBD snapshot and update its status.
-	if backup.IsReady() {
-		return nil
-	}
-
 	snapshot, err := r.createRBDSnapshot(ctx, target.poolName, target.imageName, backup)
 	if err != nil {
 		return err
@@ -1513,22 +1537,18 @@ func MakeMiddleSnapshotName(backup *mantlev1.MantleBackup, offset int) string {
 	return fmt.Sprintf("%s-offset-%d", backup.GetAnnotations()[annotRemoteUID], offset)
 }
 
-//nolint:unused // TODO: Delete this line after starting use.
-func makeVerifyImageName(target *mantlev1.MantleBackup) string {
+func MakeVerifyImageName(target *mantlev1.MantleBackup) string {
 	return mantleVerifyImagePrefix + string(target.GetUID())
 }
 
-//nolint:unused // TODO: Delete this line after starting use.
 func makeVerifyJobName(target *mantlev1.MantleBackup) string {
 	return mantleVerifyJobPrefix + string(target.GetUID())
 }
 
-//nolint:unused // TODO: Delete this line after starting use.
 func makeVerifyPVCName(target *mantlev1.MantleBackup) string {
 	return mantleVerifyPVCPrefix + string(target.GetUID())
 }
 
-//nolint:unused // TODO: Delete this line after starting use.
 func makeVerifyPVName(target *mantlev1.MantleBackup) string {
 	return mantleVerifyPVPrefix + string(target.GetUID())
 }
@@ -2363,7 +2383,6 @@ blkdiscard -z /dev/zeroout-rbd
 	return err
 }
 
-//nolint:unused // TODO: Delete this line after starting use `verify`.
 func (r *MantleBackupReconciler) createOrUpdateVerifyJob(ctx context.Context, jobName, pvcName string) error {
 	var job batchv1.Job
 	job.SetName(jobName)
@@ -2751,6 +2770,19 @@ func (r *MantleBackupReconciler) primaryCleanup(
 		return ctrl.Result{}, fmt.Errorf("failed to delete export data PVCs: %w", err)
 	}
 
+	if err := r.deleteVerifyJob(ctx, target); err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to delete verify Job: %w", err)
+	}
+	if err := r.deleteVerifyPVC(ctx, target); err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to delete verify PVC: %w", err)
+	}
+	if err := r.deleteVerifyPV(ctx, target); err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to delete verify PV: %w", err)
+	}
+	if err := r.deleteVerifyRBDImage(target); err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to delete verify RBD image: %w", err)
+	}
+
 	delete(target.GetAnnotations(), annotDiffFrom)
 	delete(target.GetAnnotations(), annotSyncMode)
 	if err := r.Update(ctx, target); err != nil {
@@ -2819,6 +2851,55 @@ func (r *MantleBackupReconciler) deleteAllExportDataPVCs(ctx context.Context, ba
 	return nil
 }
 
+func (r *MantleBackupReconciler) deleteVerifyJob(ctx context.Context, backup *mantlev1.MantleBackup) error {
+	var job batchv1.Job
+	job.SetName(makeVerifyJobName(backup))
+	job.SetNamespace(r.managedCephClusterID)
+	if err := r.Delete(ctx, &job, &client.DeleteOptions{
+		PropagationPolicy: ptr.To(metav1.DeletePropagationBackground),
+	}); err != nil && !aerrors.IsNotFound(err) {
+		return fmt.Errorf("failed to delete verify Job: %s/%s: %w", job.GetNamespace(), job.GetName(), err)
+	}
+	return nil
+}
+
+func (r *MantleBackupReconciler) deleteVerifyPV(ctx context.Context, backup *mantlev1.MantleBackup) error {
+	var pv corev1.PersistentVolume
+	pv.SetName(makeVerifyPVName(backup))
+	if err := r.Delete(ctx, &pv); err != nil && !aerrors.IsNotFound(err) {
+		return fmt.Errorf("failed to delete verify PV: %w", err)
+	}
+	return nil
+}
+
+func (r *MantleBackupReconciler) deleteVerifyPVC(ctx context.Context, backup *mantlev1.MantleBackup) error {
+	var pvc corev1.PersistentVolumeClaim
+	pvc.SetName(makeVerifyPVCName(backup))
+	pvc.SetNamespace(r.managedCephClusterID)
+	if err := r.Delete(ctx, &pvc); err != nil && !aerrors.IsNotFound(err) {
+		return fmt.Errorf("failed to delete verify PVC: %w", err)
+	}
+	return nil
+}
+
+func (r *MantleBackupReconciler) deleteVerifyRBDImage(backup *mantlev1.MantleBackup) error {
+	if backup.Status.PVManifest == "" {
+		// The RBD image for verification is created after the PV manifest is stored in the status.
+		// Thus, when the PV manifest is missing, the RBD image hasn't been created, so we can safely return nil.
+		return nil
+	}
+	var pv corev1.PersistentVolume
+	if err := json.Unmarshal([]byte(backup.Status.PVManifest), &pv); err != nil {
+		return fmt.Errorf("failed to unmarshal PV manifest: %w", err)
+	}
+	pool := pv.Spec.CSI.VolumeAttributes["pool"]
+	if pool == "" {
+		return errors.New("pool name is missing in the PV manifest")
+	}
+	image := MakeVerifyImageName(backup)
+	return deleteRBDImageAsynchronously(r.ceph, pool, image)
+}
+
 func (r *MantleBackupReconciler) secondaryCleanup(
 	ctx context.Context,
 	target *mantlev1.MantleBackup,
@@ -2866,6 +2947,19 @@ func (r *MantleBackupReconciler) secondaryCleanup(
 		return ctrl.Result{}, fmt.Errorf("failed to delete zeroout PV: %w", err)
 	}
 
+	if err := r.deleteVerifyJob(ctx, target); err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to delete verify Job: %w", err)
+	}
+	if err := r.deleteVerifyPVC(ctx, target); err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to delete verify PVC: %w", err)
+	}
+	if err := r.deleteVerifyPV(ctx, target); err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to delete verify PV: %w", err)
+	}
+	if err := r.deleteVerifyRBDImage(target); err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to delete verify RBD image: %w", err)
+	}
+
 	if deleteExportData {
 		if err := r.deleteAllExportedData(ctx, target); err != nil {
 			return ctrl.Result{}, fmt.Errorf("failed to delete exported data in the object storage: %w", err)

internal/controller/mantlebackup_controller_test.go

@@ -1591,9 +1591,9 @@ var _ = Describe("export and upload", func() {
 			origSize := *resource.NewQuantity(testutil.FakeRBDSnapshotSize-1, resource.BinarySI)
 			mbr.backupTransferPartSize = origSize
 
-			// We use reconcileAsStandalone to focus on the part size update logic.
+			// We use reconcilePre to focus on the part size update logic.
 			Eventually(func(g Gomega) {
-				result, err := mbr.reconcileAsStandalone(ctx, backup)
+				result, err := mbr.reconcilePre(ctx, backup)
 				g.Expect(result.IsZero()).To(BeTrue())
 				g.Expect(err).NotTo(HaveOccurred())
 
@@ -1607,7 +1607,7 @@ var _ = Describe("export and upload", func() {
 			mbr.backupTransferPartSize = newSize
 
 			Eventually(func(g Gomega) {
-				result, err := mbr.reconcileAsStandalone(ctx, backup)
+				result, err := mbr.reconcilePre(ctx, backup)
 				g.Expect(result.IsZero()).To(BeTrue())
 				g.Expect(err).NotTo(HaveOccurred())
 			}, "10s").Should(Succeed())
@@ -2583,7 +2583,7 @@ var _ = Describe("MantleBackupReconciler", func() {
 							"pool":          pvSrc.Spec.CSI.VolumeAttributes["pool"],
 							"staticVolume":  "true",
 						},
-						VolumeHandle: makeVerifyImageName(backup),
+						VolumeHandle: MakeVerifyImageName(backup),
 					},
 				},
 				PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain,
@@ -2739,7 +2739,7 @@ var _ = Describe("MantleBackupReconciler", func() {
 
 		It("generates correct names", func(ctx SpecContext) {
 			for _, backup := range []*mantlev1.MantleBackup{backupSuccess, backupFail} {
-				Expect(makeVerifyImageName(backup)).To(Equal(mantleVerifyImagePrefix + string(backup.GetUID())))
+				Expect(MakeVerifyImageName(backup)).To(Equal(mantleVerifyImagePrefix + string(backup.GetUID())))
 				Expect(makeVerifyJobName(backup)).To(Equal(mantleVerifyJobPrefix + string(backup.GetUID())))
 				Expect(makeVerifyPVName(backup)).To(Equal(mantleVerifyPVPrefix + string(backup.GetUID())))
 				Expect(makeVerifyPVCName(backup)).To(Equal(mantleVerifyPVCPrefix + string(backup.GetUID())))

internal/controller/mantlerestore_controller.go

@@ -29,6 +29,8 @@ type MantleRestoreReconciler struct {
 }
 
 const (
+	mbAnnotationSkipVerifyKey       = "mantle.cybozu.io/skip-verify"
+	mbAnnotationSkipVerifyValue     = "true"
 	MantleRestoreFinalizerName      = "mantlerestore.mantle.cybozu.io/finalizer"
 	RestoringPVFinalizerName        = "mantle.cybozu.io/restoring-pv-finalizer"
 	PVAnnotationRestoredBy          = "mantle.cybozu.io/restored-by"
@@ -140,6 +142,12 @@ func (r *MantleRestoreReconciler) restore(ctx context.Context, restore *mantlev1
 		return requeueReconciliation(), nil
 	}
 
+	// check if the backup is verified or verification is skipped
+	if skip, ok := backup.GetAnnotations()[mbAnnotationSkipVerifyKey]; !backup.IsVerified() && (!ok || skip != mbAnnotationSkipVerifyValue) {
+		logger.Info("verification is not completed", "backup", backup.Name, "namespace", backup.Namespace)
+		return requeueReconciliation(), nil
+	}
+
 	// create a clone image from the backup
 	if err := r.cloneImageFromBackup(ctx, restore, &backup); err != nil {
 		logger.Error(err, "failed to clone image from backup", "backup", backup.Name, "namespace", backup.Namespace)

internal/controller/persistentvolume_controller.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"slices"
 
 	"github.com/cybozu-go/mantle/internal/ceph"
 	corev1 "k8s.io/api/core/v1"
@@ -130,27 +129,5 @@ func (r *PersistentVolumeReconciler) removeRBDImage(ctx context.Context, pv *cor
 	pool := pv.Spec.CSI.VolumeAttributes["pool"]
 	logger.Info("removing image", "pool", pool, "image", image)
 
-	images, err := r.ceph.RBDLs(pool)
-	if err != nil {
-		return fmt.Errorf("failed to list RBD images: %w", err)
-	}
-
-	if !slices.Contains(images, image) {
-		return nil
-	}
-
-	imageInfo, err := r.ceph.RBDInfo(pool, image)
-	if err != nil {
-		return fmt.Errorf("failed to get info about the RBD image: %s/%s: %w", pool, image, err)
-	}
-
-	if err := r.ceph.RBDTrashMv(pool, image); err != nil {
-		return fmt.Errorf("failed to move the RBD image to trash: %s/%s: %w", pool, image, err)
-	}
-
-	if err := r.ceph.CephRBDTaskAddTrashRemove(pool, imageInfo.ID); err != nil {
-		return fmt.Errorf("failed to add task to remove the RBD image from trash: %s/%s: %w", pool, image, err)
-	}
-
-	return nil
+	return deleteRBDImageAsynchronously(r.ceph, pool, image)
 }

internal/controller/util.go

@@ -160,3 +160,29 @@ func requeueReconciliation() ctrl.Result {
 	}
 	return ctrl.Result{RequeueAfter: duration}
 }
+
+func deleteRBDImageAsynchronously(ceph ceph.CephCmd, pool, image string) error {
+	images, err := ceph.RBDLs(pool)
+	if err != nil {
+		return fmt.Errorf("failed to list RBD images: %w", err)
+	}
+
+	if !slices.Contains(images, image) {
+		return nil
+	}
+
+	imageInfo, err := ceph.RBDInfo(pool, image)
+	if err != nil {
+		return fmt.Errorf("failed to get info about the RBD image: %s/%s: %w", pool, image, err)
+	}
+
+	if err := ceph.RBDTrashMv(pool, image); err != nil {
+		return fmt.Errorf("failed to move the RBD image to trash: %s/%s: %w", pool, image, err)
+	}
+
+	if err := ceph.CephRBDTaskAddTrashRemove(pool, imageInfo.ID); err != nil {
+		return fmt.Errorf("failed to add task to remove the RBD image from trash: %s/%s: %w", pool, image, err)
+	}
+
+	return nil
+}