implement verify function

Signed-off-by: Yuji Ito <llamerada.jp@gmail.com>

Author: llamerada-jp

api/v1/mantlebackup_types.go

@@ -59,6 +59,7 @@ type MantleBackupStatus struct {
 const (
 	BackupConditionReadyToUse     = "ReadyToUse"
 	BackupConditionSyncedToRemote = "SyncedToRemote"
+	BackupConditionVerified       = "Verified"
 
 	// Reasons for ConditionReadyToUse
 	BackupReasonNone                 = "NoProblem"

internal/controller/mantlebackup_controller.go

@@ -53,6 +53,8 @@ const (
 	labelComponentExportJob       = "export-job"
 	labelComponentUploadJob       = "upload-job"
 	labelComponentImportJob       = "import-job"
+	labelComponentVerifyJob       = "verify-job"
+	labelComponentVerifyVolume    = "verify-volume"
 	labelComponentZeroOutJob      = "zeroout-job"
 	labelComponentZeroOutVolume   = "zeroout-volume"
 	annotRemoteUID                = "mantle.cybozu.io/remote-uid"
@@ -619,6 +621,115 @@ func (r *MantleBackupReconciler) replicateManifests(
 	return ctrl.Result{}, nil
 }
 
+func (r *MantleBackupReconciler) verify(
+	ctx context.Context,
+	backup *mantlev1.MantleBackup,
+) error {
+	var storedPVC corev1.PersistentVolumeClaim
+	if err := json.Unmarshal([]byte(backup.Status.PVCManifest), &storedPVC); err != nil {
+		return fmt.Errorf("failed to unmarshal PVC manifest: %w", err)
+	}
+
+	var storedPV corev1.PersistentVolume
+	if err := json.Unmarshal([]byte(backup.Status.PVManifest), &storedPV); err != nil {
+		return fmt.Errorf("failed to unmarshal PV manifest: %w", err)
+	}
+
+	snapshotName := backup.Name
+	cloneName := fmt.Sprintf("mantle-verify-%s", backup.UID)
+	pvName := fmt.Sprintf("mantle-verify-%s", backup.UID)
+	pvcName := fmt.Sprintf("mantle-verify-%s", backup.UID)
+	jobName := fmt.Sprintf("mantle-verify-%s", backup.UID)
+
+	// create a clone by the snapshot bound to the MB
+	if err := createCloneByPV(ctx, r.ceph, &storedPV, snapshotName, cloneName); err != nil {
+		return fmt.Errorf("failed to create a clone by the snapshot: %w", err)
+	}
+
+	// create a static PV with the clone
+	if err := r.createOrUpdateStaticPV(
+		ctx,
+		&storedPV,
+		cloneName,
+		corev1.ResourceList{
+			corev1.ResourceStorage: *resource.NewQuantity(*backup.Status.SnapSize, resource.BinarySI),
+		},
+		pvName,
+		labelComponentVerifyVolume,
+	); err != nil {
+		return fmt.Errorf("failed to create a static PV with the clone: %w", err)
+	}
+
+	// create a PVC with the PV
+	if err := r.createOrUpdateStaticPVC(ctx, pvcName, pvName, labelComponentVerifyVolume, storedPVC.Spec.Resources); err != nil {
+		return fmt.Errorf("failed to create a PVC with the PV: %w", err)
+	}
+
+	// create a Job to execute e2fsck on the PVC
+	if err := r.createOrUpdateVerifyJob(ctx, jobName, pvcName); err != nil {
+		return fmt.Errorf("failed to create a Job to execute e2fsck on the PVC: %w", err)
+	}
+
+	// wait for the Job to complete
+	jobFinished, jobSucceeded, err := r.checkJobStatus(ctx, jobName)
+	if err != nil {
+		return fmt.Errorf("failed to check the Job status: %w", err)
+	}
+	if !jobFinished {
+		// still running
+		return nil
+	}
+
+	// update MB's conditions field
+	condition := metav1.ConditionTrue
+	reason := ""
+	if !jobSucceeded {
+		condition = metav1.ConditionFalse
+	}
+	if err := r.updateMantleBackupCondition(
+		ctx, backup,
+		mantlev1.BackupConditionVerified,
+		condition,
+		reason,
+	); err != nil {
+		return fmt.Errorf("failed to update MantleBackup condition: %w", err)
+	}
+
+	return nil
+}
+
+// checkJobStatus checks the status of the Job with the given name.
+// It returns (is job finished, is job succeeded, error).
+func (r *MantleBackupReconciler) checkJobStatus(ctx context.Context, jobName string) (bool, bool, error) {
+	var job batchv1.Job
+	if err := r.Get(
+		ctx,
+		types.NamespacedName{
+			Namespace: r.managedCephClusterID,
+			Name:      jobName,
+		},
+		&job,
+	); err != nil {
+		if aerrors.IsNotFound(err) {
+			// The cache must be stale.
+			return false, false, nil
+		}
+		return false, false, fmt.Errorf("failed to get Job for checking job status: %w", err)
+	}
+
+	for _, c := range job.Status.Conditions {
+		if c.Type == batchv1.JobComplete && c.Status == corev1.ConditionTrue {
+			return true, true, nil
+		}
+		if c.Type == batchv1.JobFailed && c.Status == corev1.ConditionTrue {
+			return true, false, nil
+		}
+	}
+
+	// job is still running
+	return false, false, nil
+}
+
 func (r *MantleBackupReconciler) provisionRBDSnapshot(
 	ctx context.Context,
 	backup *mantlev1.MantleBackup,
@@ -2125,6 +2236,80 @@ blkdiscard -z /dev/zeroout-rbd
 	return err
 }
 
+func (r *MantleBackupReconciler) createOrUpdateVerifyJob(ctx context.Context, jobName, pvcName string) error {
+	var job batchv1.Job
+	job.SetName(jobName)
+	job.SetNamespace(r.managedCephClusterID)
+	_, err := ctrl.CreateOrUpdate(ctx, r.Client, &job, func() error {
+		labels := job.GetLabels()
+		if labels == nil {
+			labels = map[string]string{}
+		}
+		labels["app.kubernetes.io/name"] = labelAppNameValue
+		labels["app.kubernetes.io/component"] = labelComponentVerifyJob
+		job.SetLabels(labels)
+
+		backoffLimit := int32(65535)
+		job.Spec.BackoffLimit = &backoffLimit
+
+		verifyContainerName := "verify"
+		job.Spec.PodFailurePolicy = &batchv1.PodFailurePolicy{
+			Rules: []batchv1.PodFailurePolicyRule{
+				{
+					Action: "FailJob",
+					OnExitCodes: &batchv1.PodFailurePolicyOnExitCodesRequirement{
+						ContainerName: &verifyContainerName,
+						Operator:      batchv1.PodFailurePolicyOnExitCodesOpIn,
+						Values:        []int32{1, 2, 3, 4, 5, 6, 7}, // File system errors
+					},
+				},
+			},
+		}
+
+		tru := true
+		var zero int64 = 0
+		job.Spec.Template.Spec.Containers = []corev1.Container{
+			{
+				Name:  "verify",
+				Image: r.podImage,
+				Command: []string{
+					"/user/sbin/e2fsck",
+					"-fn",
+					"/dev/verify-rbd",
+				},
+				SecurityContext: &corev1.SecurityContext{
+					Privileged: &tru,
+					RunAsGroup: &zero,
+					RunAsUser:  &zero,
+				},
+				VolumeDevices: []corev1.VolumeDevice{
+					{
+						Name:       "verify-rbd",
+						DevicePath: "/dev/verify-rbd",
+					},
+				},
+				ImagePullPolicy: corev1.PullIfNotPresent,
+			},
+		}
+
+		job.Spec.Template.Spec.RestartPolicy = corev1.RestartPolicyNever
+
+		job.Spec.Template.Spec.Volumes = []corev1.Volume{
+			{
+				Name: "verify-rbd",
+				VolumeSource: corev1.VolumeSource{
+					PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
+						ClaimName: pvcName,
+					},
+				},
+			},
+		}
+
+		return nil
+	})
+	return err
+}
+
 func (r *MantleBackupReconciler) hasZeroOutJobCompleted(ctx context.Context, backup *mantlev1.MantleBackup) (bool, error) {
 	var job batchv1.Job
 	if err := r.Get(
@@ -2460,16 +2645,13 @@ func (r *MantleBackupReconciler) primaryCleanup(
 		return ctrl.Result{}, nil
 	}
 
-	// Update the status of the MantleBackup. Use Patch here because Update() is
-	// likely to fail due to "the object has been modified" error.
-	newTarget := target.DeepCopy()
-	meta.SetStatusCondition(&newTarget.Status.Conditions, metav1.Condition{
-		Type:   mantlev1.BackupConditionSyncedToRemote,
-		Status: metav1.ConditionTrue,
-		Reason: mantlev1.BackupReasonNone,
-	})
-	if err := r.Client.Status().Patch(ctx, newTarget, client.MergeFrom(target)); err != nil {
-		return ctrl.Result{}, fmt.Errorf("failed to update SyncedToRemote to True by Patch: %w", err)
+	if err := r.updateMantleBackupCondition(
+		ctx, target,
+		mantlev1.BackupConditionSyncedToRemote,
+		metav1.ConditionTrue,
+		mantlev1.BackupReasonNone,
+	); err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to set SyncedToRemote condition to True: %w", err)
 	}
 	logger.Info("succeeded to sync a backup to the remote ceph cluster")
 
@@ -2677,3 +2859,24 @@ func (r *MantleBackupReconciler) deleteMiddleSnapshots(backup *mantlev1.MantleBa
 
 	return nil
 }
+
+func (r *MantleBackupReconciler) updateMantleBackupCondition(
+	ctx context.Context,
+	target *mantlev1.MantleBackup,
+	conditionType string,
+	status metav1.ConditionStatus,
+	reason string,
+) error {
+	// Update the status of the MantleBackup. Use Patch here because Update() is
+	// likely to fail due to "the object has been modified" error.
+	willBeApplied := target.DeepCopy()
+	meta.SetStatusCondition(&willBeApplied.Status.Conditions, metav1.Condition{
+		Type:   mantlev1.BackupConditionSyncedToRemote,
+		Status: metav1.ConditionTrue,
+		Reason: mantlev1.BackupReasonNone,
+	})
+	if err := r.Client.Status().Patch(ctx, willBeApplied, client.MergeFrom(target)); err != nil {
+		return fmt.Errorf("failed to update MantleBackup condition by patch: %w", err)
+	}
+	return nil
+}