Implementing Centralized Logging

helm repo add fluent https://fluent.github.io/helm-charts
helm install fluentd fluent/fluentd -n logging --create-namespace

helm install es elastic/elasticsearch -n logging
helm install kibana elastic/kibana -n logging

<source>
  @type tail
  path /var/log/containers/*.log
  pos_file /var/log/fluentd-containers.pos
  tag kube.*
  <parse> @type json </parse>
</source>
<match kube.**>
  @type elasticsearch
  host es-master.logging
  port 9200
  logstash_format true
</match>

helm install loki grafana/loki-stack -n logging --create-namespace

scrape_configs:
- job_name: kubernetes-pods
  kubernetes_sd_configs: [{ role: pod }]
  relabel_configs:
  - { source_labels: [__meta_kubernetes_namespace], target_label: namespace }
  - { source_labels: [__meta_kubernetes_pod_label_app], target_label: app }

{namespace="prod", app="web"} |= "error" | json | line_format "{{.msg}}"
rate({app="web"} |= "5xx" [5m])
sum by (status) (count_over_time({app="web"} | json [5m]))