Renew metrics

This commit add following metrics.
- mantle_backup_duration_seconds_total
- mantle_mantlebackupconfig_info
- mantle_backup_duration_seconds_bucket
- mantle_backup_duration_seconds_sum
- mantle_backup_duration_seconds_count

Signed-off-by: Kohya Shiozaki <kouyan120706@gmail.com>
Co-authored-by: Ryotaro Banno <ryotaro.banno@gmail.com>
Co-authored-by: Yuji Ito <llamerada.jp@gmail.com>

Author: 3 people

README.md

@@ -71,6 +71,35 @@ make uninstall
 make undeploy
 ```
 
+## Prometheus metrics
+
+### `mantle_backup_duration_seconds_total`
+
+`mantle_backup_duration_seconds_total` is a Counter that indicates the time from the creationTimestamp to the completion of the backup.
+
+| Label                   | Description             |
+| ----------------------- | ----------------------- |
+| `persistentvolumeclaim` | The PVC name.           |
+| `resource_namespace`    | The resource namespace. |
+
+### `mantle_mantlebackupconfig_info`
+
+`mantle_mantlebackupconfig_info` is a Gauge that indicates information about the backup configuration.
+
+| Label                   | Description             |
+| ----------------------- | ----------------------- |
+| `persistentvolumeclaim` | The PVC name.           |
+| `resource_namespace`    | The resource namespace. |
+
+### `mantle_backup_duration_seconds`
+
+`mantle_backup_duration_seconds` is a Histogram that indicates the time from the creationTimestamp to the completion of the backup.
+
+| Label                   | Description             |
+| ----------------------- | ----------------------- |
+| `persistentvolumeclaim` | The PVC name.           |
+| `resource_namespace`    | The resource namespace. |
+
 ## Development
 
 The following tools should be installed manually.

internal/controller/mantlebackup_controller.go

@@ -15,7 +15,6 @@ import (
 	_ "embed"
 
 	mantlev1 "github.com/cybozu-go/mantle/api/v1"
-	"github.com/cybozu-go/mantle/cmd/backup"
 	"github.com/cybozu-go/mantle/internal/ceph"
 	"github.com/cybozu-go/mantle/internal/controller/internal/objectstorage"
 	"github.com/cybozu-go/mantle/internal/controller/metrics"
@@ -2446,19 +2445,15 @@ func (r *MantleBackupReconciler) primaryCleanup(
 	}
 
 	duration := time.Since(target.GetCreationTimestamp().Time).Seconds()
-	source := "none"
-	if _, ok := target.GetLabels()[backup.MantleBackupConfigUID]; ok {
-		source = "mantle-backup-config"
-	}
-	metrics.BackupCreationDuration.
-		With(prometheus.Labels{
-			"cluster_namespace": r.managedCephClusterID,
-			// PVC is located in the same namespace as the MantleBackup.
-			"pvc_namespace": target.GetNamespace(),
-			"pvc":           target.Spec.PVC,
-			"source":        source,
-		}).
-		Observe(duration)
+	metrics.BackupDurationSecondsTotal.With(prometheus.Labels{
+		"persistentvolumeclaim": target.Spec.PVC,
+		"resource_namespace":    target.GetNamespace(),
+	}).Add(float64(duration))
+
+	metrics.BackupDurationSeconds.With(prometheus.Labels{
+		"persistentvolumeclaim": target.Spec.PVC,
+		"resource_namespace":    target.GetNamespace(),
+	}).Observe(duration)
 
 	return ctrl.Result{}, nil
 }

internal/controller/mantlebackupconfig_controller.go

@@ -19,14 +19,17 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	mantlev1 "github.com/cybozu-go/mantle/api/v1"
+	"github.com/cybozu-go/mantle/internal/controller/metrics"
+	"github.com/prometheus/client_golang/prometheus"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 const (
-	MantleBackupConfigFinalizerName     = "mantlebackupconfig.mantle.cybozu.io/finalizer"
-	MantleBackupConfigCronJobNamePrefix = "mbc-"
+	MantleBackupConfigFinalizerName              = "mantlebackupconfig.mantle.cybozu.io/finalizer"
+	MantleBackupConfigAnnotationManagedClusterID = "mantlebackupconfig.mantle.cybozu.io/managed-cluster-id"
+	MantleBackupConfigCronJobNamePrefix          = "mbc-"
 )
 
 // MantleBackupConfigReconciler reconciles a MantleBackupConfig object
@@ -88,7 +91,14 @@ func (r *MantleBackupConfigReconciler) Reconcile(ctx context.Context, req ctrl.R
 
 	// When the deletionTimestamp is set, remove the finalizer and finish reconciling.
 	if !mbc.DeletionTimestamp.IsZero() {
-		if controllerutil.ContainsFinalizer(&mbc, MantleBackupConfigFinalizerName) {
+		if controllerutil.ContainsFinalizer(&mbc, MantleBackupConfigFinalizerName) &&
+			mbc.Annotations[MantleBackupConfigAnnotationManagedClusterID] == r.managedCephClusterID {
+			_ = metrics.BackupConfigInfo.Delete(prometheus.Labels{
+				"persistentvolumeclaim": mbc.Spec.PVC,
+				"resource_namespace":    mbc.Namespace,
+				"mantlebackupconfig":    mbc.Name,
+			})
+
 			// Delete the CronJob. If we failed to delete it because it's not found, ignore the error.
 			logger.Info("start deleting cronjobs")
 			if err := r.deleteCronJob(ctx, &mbc, cronJobInfo.namespace); err != nil && !errors.IsNotFound(err) {
@@ -119,12 +129,24 @@ func (r *MantleBackupConfigReconciler) Reconcile(ctx context.Context, req ctrl.R
 
 	// Set the finalizer if it's not yet set.
 	if !controllerutil.ContainsFinalizer(&mbc, MantleBackupConfigFinalizerName) {
+		if mbc.Annotations == nil {
+			mbc.Annotations = make(map[string]string)
+		}
+		mbc.Annotations[MantleBackupConfigAnnotationManagedClusterID] = r.managedCephClusterID
+
 		controllerutil.AddFinalizer(&mbc, MantleBackupConfigFinalizerName)
 		if err := r.Client.Update(ctx, &mbc); err != nil {
 			return ctrl.Result{}, fmt.Errorf("failed to add mbc finalizer: %w", err)
 		}
 	}
 
+	// Export metrics.
+	metrics.BackupConfigInfo.With(prometheus.Labels{
+		"persistentvolumeclaim": mbc.Spec.PVC,
+		"resource_namespace":    mbc.ObjectMeta.Namespace,
+		"mantlebackupconfig":    mbc.ObjectMeta.Name,
+	}).Set(1)
+
 	// Create or update the CronJob
 	if err := r.createOrUpdateCronJob(
 		ctx,

internal/controller/metrics/metrics.go

@@ -5,20 +5,46 @@ import (
 	runtimemetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 )
 
-const subsystem = "mantle"
+const namespace = "mantle"
 
 var (
-	BackupCreationDuration = prometheus.NewHistogramVec(
+	BackupDurationSecondsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: namespace,
+			Name:      "backup_duration_seconds_total",
+			Help:      "The time from the creationTimestamp to the completion of the backup.",
+		},
+		[]string{"persistentvolumeclaim", "resource_namespace"},
+	)
+
+	BackupConfigInfo = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      "mantlebackupconfig_info",
+			Help:      "Information about the backup configuration.",
+		},
+		[]string{"persistentvolumeclaim", "resource_namespace", "mantlebackupconfig"},
+	)
+
+	BackupDurationSeconds = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
-			Subsystem: subsystem,
-			Name:      "backup_creation_duration_seconds",
-			Help:      "Duration in seconds of backup creation.",
-			Buckets:   []float64{100, 250, 500, 750, 1_000, 2_500, 5_000, 7_500, 10_000, 25_000, 50_000, 75_000, 100_000, 250_000},
+			Namespace: namespace,
+			Name:      "backup_duration_seconds",
+			Help:      "The time from the creationTimestamp to the completion of the backup.",
+			Buckets:   []float64{100, 200, 400, 800, 1600, 3200, 9600, 28800, 86400, 259200},
 		},
-		[]string{"cluster_namespace", "pvc_namespace", "pvc", "source"},
+		[]string{"persistentvolumeclaim", "resource_namespace"},
 	)
 )
 
 func init() {
-	runtimemetrics.Registry.MustRegister(BackupCreationDuration)
+	metricsToRegister := []prometheus.Collector{
+		BackupDurationSecondsTotal,
+		BackupConfigInfo,
+		BackupDurationSeconds,
+	}
+
+	for _, metric := range metricsToRegister {
+		runtimemetrics.Registry.MustRegister(metric)
+	}
 }

test/e2e/multik8s/misc_test.go

@@ -14,6 +14,39 @@ import (
 	"k8s.io/apimachinery/pkg/api/meta"
 )
 
+var _ = Describe("metrics tests", func() {
+	namespace := util.GetUniqueName("ns-")
+	pvcName := util.GetUniqueName("pvc-")
+	backupName := util.GetUniqueName("mb-")
+	backupConfigName := util.GetUniqueName("mbc-")
+
+	It("should setup", func(ctx SpecContext) {
+		SetupEnvironment(namespace)
+		CreatePVC(ctx, PrimaryK8sCluster, namespace, pvcName)
+		CreateMantleBackup(PrimaryK8sCluster, namespace, pvcName, backupName)
+		WaitMantleBackupSynced(namespace, backupName)
+		CreateMantleBackupConfig(PrimaryK8sCluster, namespace, pvcName, backupConfigName)
+	})
+
+	DescribeTable("metrics should be exposed",
+		func(ctx SpecContext, metricName string) {
+			Eventually(ctx, func(g Gomega) {
+				controllerPod, err := GetControllerPodName(PrimaryK8sCluster)
+				g.Expect(err).NotTo(HaveOccurred())
+				stdout, _, err := Kubectl(PrimaryK8sCluster, nil, "exec", "-n", CephClusterNamespace, controllerPod, "--",
+					"curl", "-s", "http://localhost:8080/metrics")
+				g.Expect(err).NotTo(HaveOccurred())
+				g.Expect(strings.Contains(string(stdout), metricName)).To(BeTrue())
+			}).Should(Succeed())
+		},
+		Entry(`mantle_backup_duration_seconds_total`, `mantle_backup_duration_seconds_total`),
+		Entry(`mantle_mantlebackupconfig_info`, `mantle_mantlebackupconfig_info`),
+		Entry(`mantle_backup_duration_seconds_bucket`, `mantle_backup_duration_seconds_bucket`),
+		Entry(`mantle_backup_duration_seconds_sum`, `mantle_backup_duration_seconds_sum`),
+		Entry(`mantle_backup_duration_seconds_count`, `mantle_backup_duration_seconds_count`),
+	)
+})
+
 var _ = Describe("miscellaneous tests", func() {
 	It("should succeed to back up if backup-transfer-part-size is changed during uploading", func(ctx SpecContext) {
 		namespace := util.GetUniqueName("ns-")
@@ -348,25 +381,4 @@ spec:
 			0, // reset an import Job to the original one to revive the Job.
 		),
 	)
-
-	It("should get metrics from the controller pod in the primary cluster", func(ctx SpecContext) {
-		metrics := []string{
-			`mantle_backup_creation_duration_seconds_count`,
-			`mantle_backup_creation_duration_seconds_sum`,
-		}
-		ensureMetricsAreExposed(metrics)
-	})
 })
-
-func ensureMetricsAreExposed(metrics []string) {
-	GinkgoHelper()
-	controllerPod, err := GetControllerPodName(PrimaryK8sCluster)
-	Expect(err).NotTo(HaveOccurred())
-
-	stdout, _, err := Kubectl(PrimaryK8sCluster, nil, "exec", "-n", CephClusterNamespace, controllerPod, "--",
-		"curl", "-s", "http://localhost:8080/metrics")
-	Expect(err).NotTo(HaveOccurred())
-	for _, metric := range metrics {
-		Expect(strings.Contains(string(stdout), metric)).To(BeTrue())
-	}
-}

test/e2e/multik8s/testutil/testdata/mantlebackupconfig-template.yaml

@@ -0,0 +1,10 @@
+apiVersion: mantle.cybozu.io/v1
+kind: MantleBackupConfig
+metadata:
+  name: %s
+  namespace: %s
+spec:
+  pvc: %s
+  expire: "1d"
+  schedule: "00 00 * * *"
+  suspend: false

test/e2e/multik8s/testutil/util.go

@@ -46,6 +46,8 @@ var (
 	testRBDPoolSCTemplate string
 	//go:embed testdata/mantlebackup-template.yaml
 	testMantleBackupTemplate string
+	//go:embed testdata/mantlebackupconfig-template.yaml
+	testMantleBackupConfigTemplate string
 	//go:embed testdata/mantlerestore-template.yaml
 	testMantleRestoreTemplate string
 	//go:embed testdata/mount-deploy-template.yaml
@@ -137,6 +139,15 @@ func ApplyMantleBackupTemplate(clusterNo int, namespace, pvcName, backupName str
 	return nil
 }
 
+func ApplyMantleBackupConfigTemplate(clusterNo int, namespace, pvcName, backupConfigName string) error {
+	manifest := fmt.Sprintf(testMantleBackupConfigTemplate, backupConfigName, namespace, pvcName)
+	_, _, err := Kubectl(clusterNo, []byte(manifest), "apply", "-f", "-")
+	if err != nil {
+		return fmt.Errorf("kubectl apply mantlebackupconfig failed. err: %w", err)
+	}
+	return nil
+}
+
 func ApplyMantleRestoreTemplate(clusterNo int, namespace, restoreName, backupName string) error {
 	manifest := fmt.Sprintf(testMantleRestoreTemplate, restoreName, restoreName, namespace, backupName)
 	_, _, err := Kubectl(clusterNo, []byte(manifest), "apply", "-f", "-")
@@ -487,6 +498,14 @@ func CreateMantleBackup(cluster int, namespace, pvcName, backupName string) {
 	}).Should(Succeed())
 }
 
+func CreateMantleBackupConfig(cluster int, namespace, pvcName, backupConfigName string) {
+	GinkgoHelper()
+	By("creating a MantleBackupConfig object")
+	Eventually(func() error {
+		return ApplyMantleBackupConfigTemplate(cluster, namespace, pvcName, backupConfigName)
+	}).Should(Succeed())
+}
+
 func WaitMantleBackupReadyToUse(cluster int, namespace, backupName string) {
 	GinkgoHelper()
 	By("checking MantleBackup's ReadyToUse status")