Configure Alert Receivers

APP_NAMESPACE=my-namespace
PUSHOVER_USER_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
PUSHOVER_API_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx

kubectl -n "${APP_NAMESPACE}" apply -f - <<YAML
apiVersion: v1
kind: Secret
metadata:
  name: pushover-user-key
type: Opaque
stringData:
  value: ${PUSHOVER_USER_KEY}
YAML

kubectl -n "${APP_NAMESPACE}" apply -f - <<YAML
apiVersion: v1
kind: Secret
metadata:
  name: pushover-api-key
type: Opaque
stringData:
  value: ${PUSHOVER_API_KEY}
YAML

kubectl -n "${APP_NAMESPACE}" apply -f - <<YAML
apiVersion: monitoring.coreos.com/v1beta1
kind: AlertmanagerConfig
metadata:
  name: ${APP_NAMESPACE}-alert-config
spec:
  receivers:
  - name: pushover
    pushoverConfigs:
    - userKey:
        key: value
        name: pushover-user-key
      token:
        key: value
        name: pushover-api-key
  route:
    receiver: pushover
    routes:
    - match:
        severity: critical
      continue: true
YAML

APP_NAMESPACE=my-namespace

TOKEN=<TOKEN> (1)
kubectl create secret generic telegram-alert-bot "--from-literal=token=${TOKEN}"

curl -X GET "https://api.telegram.org/bot${TOKEN}/getUpdates" | jq '.result[].message | select(. != null)'

CHAT_ID=<CHAT_ID>

kubectl -n "${APP_NAMESPACE}" apply -f - <<YAML
apiVersion: monitoring.coreos.com/v1beta1
kind: AlertmanagerConfig
metadata:
  name: telegram-alert-config
spec:
  receivers:
  - name: default
    telegramConfigs:
    - apiURL: https://api.telegram.org
      botToken: (1)
        key: token
        name: telegram-alert-bot
      chatID: ${CHAT_ID}
      parseMode: HTML (2)
  route:
    groupBy: [alertname, instance]
    receiver: default
YAML

APP_NAMESPACE=my-namespace

URL=<URL>
kubectl create secret generic slack-alert-webhook "--from-literal=url=${URL}"

kubectl -n "${APP_NAMESPACE}" apply -f - <<YAML
apiVersion: monitoring.coreos.com/v1beta1
kind: AlertmanagerConfig
metadata:
  name: slack-alert-config
spec:
  receivers:
  - name: default
    slackConfigs:
    - apiURL: (1)
        key: url
        name: slack-alert-webhook
      channel: '#alerts' (2)
  route:
    groupBy: [alertname, instance]
    receiver: default
YAML

APP_NAMESPACE=my-namespace

PASSWORD=<PASSWORD>
kubectl create secret generic email-alert-auth "--from-literal=password=${PASSWORD}"

kubectl -n "${APP_NAMESPACE}" apply -f - <<YAML
apiVersion: monitoring.coreos.com/v1beta1
kind: AlertmanagerConfig
metadata:
  name: alert-config
spec:
  receivers:
  - name: default
    emailConfigs:
    - to: alerts@example.com (1)
      from: my-app@example.com (2)
      smarthost: smtp.example.com:587 (3)
      authUsername: my-app@example.com (4)
      authIdentity: my-app@example.com (5)
      authPassword: (6)
        key: password
        name: email-alert-auth
  route:
    groupBy: [alertname, instance]
    receiver: default
YAML

APP_NAMESPACE=my-namespace
APPS_DOMAIN=apps.ZONE # Replace with your cluster's default app domain from https://portal.appuio.cloud/zones

kubectl -n "${APP_NAMESPACE}" create -f - <<YAML
apiVersion: v1
kind: ConfigMap
metadata:
  name: logging-webhook
data:
  logging_server.go: |
    package main

    import (
      "fmt"
      "io"
      "net/http"
      "os"
    )

    func main() {
      http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
        defer w.WriteHeader(http.StatusNoContent)
        defer r.Body.Close()

        fmt.Println(r.Method, r.URL)
        r.Header.Write(os.Stdout)
        fmt.Print("\n")
        io.Copy(os.Stdout, r.Body)
        fmt.Print("\n\n")
      })
      panic(http.ListenAndServe(":8080", nil))
    }
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: logging-webhook
  name: logging-webhook
spec:
  replicas: 1
  selector:
    matchLabels:
      app: logging-webhook
  strategy: {}
  template:
    metadata:
      creationTimestamp: null
      labels:
        app: logging-webhook
    spec:
      containers:
      - image: golang
        name: golang
        env: [{ name: 'HOME', value: '/tmp' }]
        ports:
        - containerPort: 8080
        resources: {}
        command: [go, run, "/s/logging_server.go"]
        volumeMounts:
        - mountPath: /s
          name: logging-webhook
      volumes:
      - configMap:
          name: logging-webhook
        name: logging-webhook
YAML

kubectl -n "${APP_NAMESPACE}" expose deployment logging-webhook
kubectl -n "${APP_NAMESPACE}" create ingress logging-webhook "--rule=logging-webhook-${APP_NAMESPACE}.${APPS_DOMAIN}/*=logging-webhook:8080,tls=logging-webhook-tls"
kubectl -n "${APP_NAMESPACE}" annotate ingress logging-webhook "cert-manager.io/cluster-issuer=letsencrypt-production"

kubectl -n "${APP_NAMESPACE}" apply -f - <<YAML
apiVersion: monitoring.coreos.com/v1beta1
kind: AlertmanagerConfig
metadata:
  name: debug-alert-config
spec:
  receivers:
  - name: default
    webhookConfigs:
    - url: https://logging-webhook-${APP_NAMESPACE}.${APPS_DOMAIN}/
      sendResolved: true
      httpConfig:
        authorization:
          credentials:
            name: email-alert-auth (1)
            key: password
  route:
    groupBy: [alertname, instance]
    receiver: default
YAML

curl -v https://logging-webhook-${APP_NAMESPACE}.${APPS_DOMAIN}/

kubectl -n "${APP_NAMESPACE}" logs deployment/logging-webhook -f

kubectl -n "${APP_NAMESPACE}" delete alertmanagerconfig debug-alert-config
kubectl -n "${APP_NAMESPACE}" delete ingress logging-webhook
kubectl -n "${APP_NAMESPACE}" delete service logging-webhook
kubectl -n "${APP_NAMESPACE}" delete deployment logging-webhook
kubectl -n "${APP_NAMESPACE}" delete configmap logging-webhook
kubectl -n "${APP_NAMESPACE}" delete secret logging-webhook-tls