Monitoring

Prometheus Support test cases

1. Install the Prometheus Operator (include a role and service account for it). For example:

   apiVersion: rbac.authorization.k8s.io/v1
   kind: ClusterRoleBinding
   metadata:
   name: prometheus-operator
   namespace: default
   roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
   name: prometheus-operator
   subjects:
     - kind: ServiceAccount
   name: prometheus-operator
   namespace: default
   –
   apiVersion: rbac.authorization.k8s.io/v1
   kind: ClusterRole
   metadata:
   name: prometheus-operator
   namespace: default
   rules:
     - apiGroups:
         - extensions
   resources:
         - thirdpartyresources
   verbs: [""]
     - apiGroups:
         - apiextensions.k8s.io
   resources:
         - customresourcedefinitions
   verbs: [""]
     - apiGroups:
         - monitoring.coreos.com
   resources:
         - alertmanagers
         - prometheuses
         - prometheuses/finalizers
         - servicemonitors
         - prometheusrules
         - podmonitors
   verbs: [""]
     - apiGroups:
         - apps
   resources:
         - statefulsets
   verbs: [""]
     - apiGroups: [""]
   resources:
         - configmaps
         - secrets
   verbs: ["*"]
     - apiGroups: [""]
   resources:
         - pods
   verbs: [“list”, “delete”]
     - apiGroups: [""]
   resources:
         - services
         - endpoints
   verbs: [“get”, “create”, “update”]
     - apiGroups: [""]
   resources:
         - nodes
   verbs: [“list”, “watch”]
     - apiGroups: [""]
   resources:
         - namespaces
   verbs: [“list”, “watch”]

apiVersion: v1
kind: ServiceAccount
metadata:
  name: prometheus-operator
  namespace: default
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: prometheus-operator
  name: prometheus-operator
  namespace: default
spec:
  replicas: 1
  selector:
    matchLabels:
      app: prometheus-operator
  template:
    metadata:
      labels:
        app: prometheus-operator
    spec:
      containers:
        - args:
            - --kubelet-service=kube-system/kubelet
            - --config-reloader-image=quay.io/coreos/configmap-reload:v0.0.1
          image: quay.io/coreos/prometheus-operator:v0.36.0
          name: prometheus-operator
          ports:
            - containerPort: 8080
              name: http
          resources:
            limits:
              cpu: 200m
              memory: 100Mi
            requests:
              cpu: 100m
              memory: 50Mi
      securityContext:
        runAsNonRoot: true
        runAsUser: 65534
      serviceAccountName: prometheus-operator

2. Install a Service Monitor pointing to longhon-backend service by selecting app: longhorn-manager label. For example:

   apiVersion: monitoring.coreos.com/v1
   kind: ServiceMonitor
   metadata:
   name: longhorn-backend
   labels:
   team: backend
   spec:
   selector:
   matchLabels:
   app: longhorn-manager
   namespaceSelector:
   matchNames:
       - longhorn-system
   endpoints:
     - port: manager

3. Install Prometheus (include a role and service account for it). Include the above service monitor in the Prometheus’s config. Expose to the Prometheus instance to outside using a service of type NodePort. For example:

   apiVersion: v1
   kind: ServiceAccount
   metadata:
   name: prometheus
   —
   apiVersion: rbac.authorization.k8s.io/v1
   kind: ClusterRole
   metadata:
   name: prometheus
   rules:
     - apiGroups: [""]
   resources:
         - nodes
         - services
         - endpoints
         - pods
       verbs: [“get”, “list”, “watch”]
     - apiGroups: [""]
   resources:
         - configmaps
   verbs: [“get”]
     - nonResourceURLs: ["/metrics", “/federate”]
   verbs: [“get”]
   —
   apiVersion: rbac.authorization.k8s.io/v1
   kind: ClusterRoleBinding
   metadata:
   name: prometheus
   roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
   name: prometheus
   subjects:
     - kind: ServiceAccount
   name: prometheus
   namespace: default
   —
   apiVersion: monitoring.coreos.com/v1
   kind: Prometheus
   metadata:
   name: prometheus
   spec:
   serviceAccountName: prometheus
   serviceMonitorSelector:
   matchLabels:
   team: backend
   —
   apiVersion: v1
   kind: Service
   metadata:
   name: prometheus
   spec:
   type: NodePort
   ports:
     - name: web
   port: 9090
   protocol: TCP
   targetPort: web
   ports:
       - port: 9090
   selector:
   prometheus: prometheus

4. Find the prometheus service and access Prometheus web UI using the nodeIP and the port

Note: More details can be found on https://longhorn.io/docs/1.2.2/monitoring/