How do I send audit logs to webhook in minikube

Question

In minikube, API server fails to connect to my audit log webhook, I see the following error in the api-server logs

E0308 08:30:26.457841       1 metrics.go:109] Error in audit plugin 'webhook' affecting 400 audit events: Post "http://ca-audit.armo-system:8888/": dial tcp: lookup ca-audit.armo-system on 10.42.4.254:53: no such host

I dont know why the api-server is connecting to 10.42.4.254:53 since my service ip is different:

$ kubectl -n armo-system get services
NAME       TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)    AGE
ca-audit   ClusterIP   10.109.132.114   <none>        8888/TCP   8m33s

I do not understand what I'm doing wrong, any suggestions?

---

This is how I configured my audit policy, webhook and minikube-

I pre-configured my minikube as following:

# Create the webhook config and audit policy
export C_CONTEXT=$(kubectl config current-context)
export C_NAME=$(kubectl config get-contexts ${C_CONTEXT} --no-headers | awk '{print $2}')
export C_CLUSTER=$(kubectl config get-contexts ${C_CONTEXT} --no-headers | awk '{print $3}')
export C_USER=$(kubectl config get-contexts ${C_CONTEXT} --no-headers | awk '{print $4}')

export ARMO_NAMESPACE="armo-system"
export ARMO_AUDIT_SERVICE="ca-audit"
export ARMO_AUDIT_PORT=8888

mkdir -p ~/.minikube/files/etc/ssl/certs

cat <<EOF > ~/.minikube/files/etc/ssl/certs/audit-webhook.yaml
{
  "apiVersion": "v1",
  "clusters": [
    {
      "cluster": {
        "server": "http://${ARMO_AUDIT_SERVICE}.${ARMO_NAMESPACE}:${ARMO_AUDIT_PORT}/"
      },
      "name": "${C_NAME}"
    }
  ],
  "contexts": [
    {
      "context": {
        "cluster": "${C_CLUSTER}",
        "user": "${C_USER}"
      },
      "name": "${C_NAME}"
    }
  ],
  "current-context": "${C_CONTEXT}",
  "kind": "Config",
  "preferences": {},
  "users": []
}
EOF

cat <<EOF > ~/.minikube/files/etc/ssl/certs/audit-policy.yaml
{
  "apiVersion": "audit.k8s.io/v1",
  "kind": "Policy",
  "rules": [
    {
      "level": "Metadata"
    }
  ]
}
EOF

# Copy the audit policy to `/etc/ssl/certs/.` 
sudo cp ~/.minikube/files/etc/ssl/certs/audit-policy.yaml ~/.minikube/files/etc/ssl/certs/audit-webhook.yaml /etc/ssl/certs/.

# Start the minikube, add the flags `--extra-config=apiserver.audit-policy-file=/etc/ssl/certs/audit-policy.yaml`, `--extra-config=apiserver.audit-webhook-config-file=/etc/ssl/certs/audit-webhook.yaml`
sudo -E minikube start --vm-driver=none --extra-config=apiserver.audit-policy-file=/etc/ssl/certs/audit-policy.yaml --extra-config=apiserver.audit-webhook-config-file=/etc/ssl/certs/audit-webhook.yaml

Now that my minikube is up and running, I created the namespace, service and webhook deployment:

cat <<EOF | kubectl apply -f -
---
apiVersion: v1
kind: Namespace
metadata:
  name: ${ARMO_NAMESPACE}

---
kind: Service
apiVersion: v1
metadata:
  labels:
    app: ${ARMO_AUDIT_SERVICE}
  name: ${ARMO_AUDIT_SERVICE}
  namespace: ${ARMO_NAMESPACE}
spec:
  ports:
  - port: ${ARMO_AUDIT_PORT}
    targetPort: ${ARMO_AUDIT_PORT}
    protocol: TCP
  selector:
    app: ${ARMO_AUDIT_SERVICE}
---
apiVersion: apps/v1
kind: Deployment
metadata:
    name: ${ARMO_AUDIT_SERVICE}
    namespace: ${ARMO_NAMESPACE}
    labels:
      app: ${ARMO_AUDIT_SERVICE}
spec:
  selector:
    matchLabels:
      app: ${ARMO_AUDIT_SERVICE}
  replicas: 1
  template:
    metadata:
      labels:
        app: ${ARMO_AUDIT_SERVICE}
    spec:
      containers:
        - name: ${ARMO_AUDIT_SERVICE}
          image: quay.io/armosec/k8s-ca-auditlog-ubi:dummy
          imagePullPolicy: Always
          env:
          - name: ARMO_AUDIT_PORT
            value: "${ARMO_AUDIT_PORT}"
          ports:
          - containerPort: ${ARMO_AUDIT_PORT}
            name: ${ARMO_AUDIT_SERVICE}
EOF

The webhook image code (quay.io/armosec/k8s-ca-auditlog-ubi:dummy) is as following:

package main

import (
  "encoding/json"
  "flag"
  "fmt"
  "net/http"
  "os"

  "k8s.io/apiserver/pkg/apis/audit"

  "github.com/golang/glog"
)

func main() {
  flag.Parse()
  flag.Set("alsologtostderr", "1") // display logs in stdout

  InitServer()

}

// InitServer - Initialize webhook listener
func InitServer() {
  port, ok := os.LookupEnv("ARMO_AUDIT_PORT")
  if !ok {
    port = "8888"
  }
  glog.Infof("Webhook listening on port: %s, path: %s", port, "/")
  http.HandleFunc("/", HandleRequest)
  glog.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", port), nil))
}

//HandleRequest -
func HandleRequest(w http.ResponseWriter, req *http.Request) {
  eventList := audit.EventList{}
  err := json.NewDecoder(req.Body).Decode(&eventList)
  if err != nil {
    e := fmt.Errorf("failed parsing api-server request, reason: %s", err.Error())
    glog.Errorf(e.Error())
    http.Error(w, e.Error(), http.StatusBadRequest)
    return
  }
  glog.Infof("webhook received audit list, len: %d", len(eventList.Items))
  for _, event := range eventList.Items {
    bEvent, _ := json.Marshal(event)
    glog.Infof("Received event: %s", string(bEvent))
  }

  w.WriteHeader(http.StatusOK)
}

Answer 1

Actually I don't know well about minikube but I have used audit for k8s so my answer wouldn't be help for minikube environment.

First, k8s dns format is incorrect. It should be {service-name}.{namespace}.svc or {service-name}.{namespace}.svc.cluster.local. Of course, http:// or https:// can be added also.

Second, you should give dnsPolicy option kube-apiserver.yaml. To make kube-apiserver search dns resolution in cluster, 'dnsPolicy' should be written.

sudo vi /etc/kubernetes/manifests/kube-apiserver.yaml

open kube-apiserver.yaml and add dnsPolicy: ClusterFirstWithHostNet to it . The location of dnsPolicy is below .spec and same level with containers or volumes