Prometheus on Kubernetes Cluster 설치

Kubernetes Cluster에 Prometheus 모니터링을 적용하기 위해서는 다음과 같은 리소스가 필요합니다.

• Namespace(선택사항) : 별도 네임스페이스 구성(ex. Monitoring)
• Cluster-role
• Config-map
• Deployment
• Service
• Daemonset(node-exporter)

 

---

배포하기

$ kubectl apply -f prometheus-namespace.yaml
$ kubectl apply -f prometheus-cluster-role.yaml
$ kubectl apply -f prometheus-config-map.yaml
$ kubectl apply -f prometheus-deployment.yaml
$ kubectl apply -f prometheus-node-exporter.yaml
$ kubectl apply -f prometheus-svc.yaml

 

---

 

Namespace

# prometheus-namespace.yaml

apiVersion: v1 kind: Namespace metadata:   name: monitoring

 

 

Cluster-role & roleBinding

# prometheus-cluster-role.yaml

apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata:   name: prometheus   namespace: monitoring rules: - apiGroups: [""]   resources:   - nodes   - nodes/proxy   - services   - endpoints   - pods   verbs: ["get", "list", "watch"] - apiGroups:   - extensions   resources:   - ingresses   verbs: ["get", "list", "watch"] - nonResourceURLs: ["/metrics"]   verbs: ["get"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata:   name: prometheus roleRef:   apiGroup: rbac.authorization.k8s.io   kind: ClusterRole   name: prometheus subjects: - kind: ServiceAccount   name: default   namespace: monitoring

 

 

ConfigMap

# prometheus-config-map.yaml

apiVersion: v1 kind: ConfigMap metadata:   name: prometheus-server-conf   labels:     name: prometheus-server-conf   namespace: monitoring data:   prometheus.rules: |-     groups:     - name: container memory alert       rules:       - alert: container memory usage rate is very high( > 55%)         expr: sum(container_memory_working_set_bytes{pod!="", name=""})/ sum (kube_node_status_allocatable_memory_bytes) * 100 > 55         for: 1m         labels:           severity: fatal         annotations:           summary: High Memory Usage on           identifier: ""           description: " Memory Usage: "     - name: container CPU alert       rules:       - alert: container CPU usage rate is very high( > 10%)         expr: sum (rate (container_cpu_usage_seconds_total{pod!=""}[1m])) / sum (machine_cpu_cores) * 100 > 10         for: 1m         labels:           severity: fatal         annotations:           summary: High Cpu Usage   prometheus.yml: |-     global:       scrape_interval: 5s       evaluation_interval: 5s     rule_files:       - /etc/prometheus/prometheus.rules     alerting:       alertmanagers:       - scheme: http         static_configs:         - targets:           - "alertmanager.monitoring.svc:9093"     scrape_configs:       - job_name: 'kubernetes-apiservers'         kubernetes_sd_configs:         - role: endpoints         scheme: https         tls_config:           ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt         bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token         relabel_configs:         - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]           action: keep           regex: default;kubernetes;https       - job_name: 'kubernetes-nodes'         scheme: https         tls_config:           ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt         bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token         kubernetes_sd_configs:         - role: node         relabel_configs:         - action: labelmap           regex: __meta_kubernetes_node_label_(.+)         - target_label: __address__           replacement: kubernetes.default.svc:443         - source_labels: [__meta_kubernetes_node_name]           regex: (.+)           target_label: __metrics_path__           replacement: /api/v1/nodes/${1}/proxy/metrics       - job_name: 'kubernetes-pods'         kubernetes_sd_configs:         - role: pod         relabel_configs:         - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]           action: keep           regex: true         - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]           action: replace           target_label: __metrics_path__           regex: (.+)         - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]           action: replace           regex: ([^:]+)(?::\d+)?;(\d+)           replacement: $1:$2           target_label: __address__         - action: labelmap           regex: __meta_kubernetes_pod_label_(.+)         - source_labels: [__meta_kubernetes_namespace]           action: replace           target_label: kubernetes_namespace         - source_labels: [__meta_kubernetes_pod_name]           action: replace           target_label: kubernetes_pod_name       - job_name: 'kube-state-metrics'         static_configs:           - targets: ['kube-state-metrics.kube-system.svc.cluster.local:8080']       - job_name: 'kubernetes-cadvisor'         scheme: https         tls_config:           ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt         bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token         kubernetes_sd_configs:         - role: node         relabel_configs:         - action: labelmap           regex: __meta_kubernetes_node_label_(.+)         - target_label: __address__           replacement: kubernetes.default.svc:443         - source_labels: [__meta_kubernetes_node_name]           regex: (.+)           target_label: __metrics_path__           replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor       - job_name: 'kubernetes-service-endpoints'         kubernetes_sd_configs:         - role: endpoints         relabel_configs:         - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]           action: keep           regex: true         - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]           action: replace           target_label: __scheme__           regex: (https?)         - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]           action: replace           target_label: __metrics_path__           regex: (.+)         - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]           action: replace           target_label: __address__           regex: ([^:]+)(?::\d+)?;(\d+)           replacement: $1:$2         - action: labelmap           regex: __meta_kubernetes_service_label_(.+)         - source_labels: [__meta_kubernetes_namespace]           action: replace           target_label: kubernetes_namespace         - source_labels: [__meta_kubernetes_service_name]           action: replace           target_label: kubernetes_name

 

 

Deployment

# prometheus-deployment.yaml

apiVersion: apps/v1 kind: Deployment metadata:   name: prometheus-deployment   namespace: monitoring spec:   replicas: 1   selector:     matchLabels:       app: prometheus-server   template:     metadata:       labels:         app: prometheus-server     spec:       containers:         - name: prometheus           image: prom/prometheus:latest           args:             - "--config.file=/etc/prometheus/prometheus.yml"             - "--storage.tsdb.path=/prometheus/"           ports:             - containerPort: 9090           volumeMounts:             - name: prometheus-config-volume               mountPath: /etc/prometheus/             - name: prometheus-storage-volume               mountPath: /prometheus/       volumes:         - name: prometheus-config-volume           configMap:             defaultMode: 420             name: prometheus-server-conf         - name: prometheus-storage-volume           emptyDir: {}

※ 영구 저장이 필요한 경우, prometheus-storage-volume에 PV를 연동

 

 

Node Exporter

# prometheus-node-exporter.yaml

apiVersion: apps/v1 kind: DaemonSet metadata:   name: node-exporter   namespace: monitoring   labels:     k8s-app: node-exporter spec:   selector:     matchLabels:       k8s-app: node-exporter   template:     metadata:       labels:         k8s-app: node-exporter     spec:       containers:       - image: prom/node-exporter         name: node-exporter         ports:         - containerPort: 9100           protocol: TCP           name: http --- apiVersion: v1 kind: Service metadata:   labels:     k8s-app: node-exporter   name: node-exporter   namespace: kube-system spec:   ports:   - name: http     port: 9100     nodePort: 31672     protocol: TCP   type: NodePort   selector:     k8s-app: node-exporter

 

Service

# prometheus-svc.yaml

apiVersion: v1 kind: Service metadata:   name: prometheus-service   namespace: monitoring   annotations:       prometheus.io/scrape: 'true'       prometheus.io/port:   '9090' spec:   selector:     app: prometheus-server   type: NodePort   ports:     - port: 8080       targetPort: 9090       nodePort: 30003

 

 

---