controller: record exported diff size metric during backup

Set backup_exported_diff_size_bytes on the primary by summing the sizes
of all uploaded diff parts, obtained via HeadObject against the object
storage. The sum is recomputed and Set() on every reconcile, so the value
is idempotent and needs no persistence in MantleBackup.status.

The gauge is set from two places:

  - startUpload, so the value is updated as each diff part finishes
    uploading during normal operation.
  - the beginning of reconcileAsPrimary, so the value is restored on the
    first reconcile after a controller restart. startUpload is not always
    reached because the replicate path may return early, whereas the
    beginning of reconcileAsPrimary is always reached.

The uploaded parts stay in the object storage until the secondary cluster
cleans them up after import, which happens after the backup completes, so
HeadObject always succeeds while the backup is in progress. Unlike the
secondary, the primary does not initialize objectStorageClient elsewhere,
so prepare it lazily before HeadObject.

The metric is deleted when the MantleBackup is deleted, rather than when
SyncedToRemote becomes True, to avoid removing it before Prometheus
scrapes it. DeletePartialMatch is used because the sync_mode label value
may already have been removed by the cleanup that runs beforehand.

Signed-off-by: Kohya Shiozaki <kouyan120706@gmail.com>

Author: skoya76

internal/controller/mantlebackup_controller.go

@@ -529,6 +529,18 @@ func (r *MantleBackupReconciler) reconcileAsPrimary(ctx context.Context, backup
 		return ctrl.Result{}, nil
 	}
 
+	// Restore the exported-diff-size metric so that the gauge holds the correct value
+	// even right after the controller restarts. startUpload also sets it, but the
+	// replicate path below may return early before reaching startUpload, so we set it
+	// here, which is always reached, to guarantee restoration on the first reconcile.
+	largestCompletedUploadPartNum, err := r.getLargestCompletedUploadPartNum(ctx, backup)
+	if err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to get largest completed upload part number: %w", err)
+	}
+	if err := r.setExportedDiffSizeMetric(ctx, backup, largestCompletedUploadPartNum); err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to set exported diff size metric: %w", err)
+	}
+
 	if !backup.IsVerifiedTrue() && !backup.IsVerifiedFalse() {
 		if err := r.verify(ctx, backup); err != nil {
 			return ctrl.Result{}, err
@@ -1020,6 +1032,15 @@ func (r *MantleBackupReconciler) finalizeStandalone(
 		return ctrl.Result{}, err
 	}
 
+	// Delete the metric by partial match on the stable labels, because the sync_mode
+	// label value (set from the mantle.cybozu.io/sync-mode annotation) may already
+	// have been removed by the cleanup above.
+	metrics.BackupExportedDiffSizeBytes.DeletePartialMatch(prometheus.Labels{
+		"persistentvolumeclaim": backup.Spec.PVC,
+		"resource_namespace":    backup.GetNamespace(),
+		"mantlebackup":          backup.GetName(),
+	})
+
 	controllerutil.RemoveFinalizer(backup, MantleBackupFinalizerName)
 	if err := r.Update(ctx, backup); err != nil {
 		logger.Error(err, "failed to remove finalizer", "finalizer", MantleBackupFinalizerName)
@@ -1539,6 +1560,10 @@ func (r *MantleBackupReconciler) startUpload(ctx context.Context, targetBackup *
 		return fmt.Errorf("failed to handle completed upload jobs: %w", err)
 	}
 
+	if err := r.setExportedDiffSizeMetric(ctx, targetBackup, largestCompletedUploadPartNum); err != nil {
+		return fmt.Errorf("failed to set exported diff size metric: %w", err)
+	}
+
 	if err := r.createOrUpdateUploadJobs(
 		ctx,
 		targetBackup,
@@ -1551,6 +1576,83 @@ func (r *MantleBackupReconciler) startUpload(ctx context.Context, targetBackup *
 	return nil
 }
 
+// setExportedDiffSizeMetric sets the backup_exported_diff_size_bytes gauge to the
+// total size of all diff parts uploaded so far. The sizes are obtained by
+// HeadObject-ing each part in the object storage every time, so the gauge converges
+// to the correct value no matter how many times reconciliation runs (idempotent) and
+// can be restored even after the controller restarts without persisting anything to
+// MantleBackup.status. The uploaded parts remain in the object storage until the
+// backup completes and the secondary cluster cleans them up, so HeadObject always
+// succeeds while the backup is in progress.
+func (r *MantleBackupReconciler) setExportedDiffSizeMetric(
+	ctx context.Context,
+	backup *mantlev1.MantleBackup,
+	largestCompletedUploadPartNum int,
+) error {
+	if largestCompletedUploadPartNum < 0 {
+		return nil
+	}
+
+	// The primary controller does not initialize objectStorageClient elsewhere
+	// (only the secondary path does), so initialize it here before HeadObject.
+	if err := r.prepareObjectStorageClient(ctx); err != nil {
+		return fmt.Errorf("failed to prepare object storage client: %w", err)
+	}
+
+	totalSize := int64(0)
+	for partNum := 0; partNum <= largestCompletedUploadPartNum; partNum++ {
+		key := MakeObjectNameOfExportedData(backup.GetName(), string(backup.GetUID()), partNum)
+		size, err := r.objectStorageClient.GetSize(ctx, key)
+		if err != nil {
+			return fmt.Errorf("failed to get size of exported data part %d: %w", partNum, err)
+		}
+		totalSize += size
+	}
+
+	metrics.BackupExportedDiffSizeBytes.WithLabelValues(
+		backup.Spec.PVC,
+		backup.GetNamespace(),
+		backup.GetName(),
+		backup.GetAnnotations()[annotSyncMode],
+	).Set(float64(totalSize))
+
+	return nil
+}
+
+// getLargestCompletedUploadPartNum returns the largest part number among the
+// completed upload Jobs of the backup, or -1 if there is none. Unlike
+// handleCompletedUploadJobs, it does not delete any Job, so it is safe to call from
+// the beginning of reconciliation to restore the exported-diff-size metric.
+func (r *MantleBackupReconciler) getLargestCompletedUploadPartNum(
+	ctx context.Context,
+	backup *mantlev1.MantleBackup,
+) (int, error) {
+	var jobList batchv1.JobList
+	if err := r.List(ctx, &jobList, &client.ListOptions{
+		Namespace: r.managedCephClusterID,
+		LabelSelector: labels.SelectorFromSet(map[string]string{
+			"app.kubernetes.io/name":      labelAppNameValue,
+			"app.kubernetes.io/component": labelComponentUploadJob,
+		}),
+	}); err != nil {
+		return -1, fmt.Errorf("failed to list upload Jobs: %w", err)
+	}
+
+	largestPartNum := -1
+	for _, job := range jobList.Items {
+		if !IsJobConditionTrue(job.Status.Conditions, batchv1.JobComplete) {
+			continue
+		}
+		partNum, ok := ExtractPartNumFromUploadJobName(job.GetName(), backup)
+		if !ok {
+			continue
+		}
+		largestPartNum = max(largestPartNum, partNum)
+	}
+
+	return largestPartNum, nil
+}
+
 func (r *MantleBackupReconciler) handleCompletedUploadJobs(ctx context.Context, backup *mantlev1.MantleBackup) (int, error) {
 	hook := func(partNum int) error {
 		pvc := corev1.PersistentVolumeClaim{}

internal/controller/mantlebackup_controller_test.go

@@ -1556,6 +1556,13 @@ var _ = Describe("export and upload", func() {
 		)
 		mbr.ceph = testutil.NewFakeRBD()
 
+		// The exported-diff-size metric HeadObjects each uploaded part via the
+		// object storage client. Stub it so the upload flow can run in tests.
+		mockObjectStorage := objectstorage.NewMockBucket(mockCtrl)
+		mockObjectStorage.EXPECT().GetSize(gomock.Any(), gomock.Any()).
+			Return(int64(0), nil).AnyTimes()
+		mbr.objectStorageClient = mockObjectStorage
+
 		ns = resMgr.CreateNamespace()
 	})