diff --git a/flux.yml b/flux.yml index e6fc527..a3efb60 100644 --- a/flux.yml +++ b/flux.yml @@ -75,9 +75,9 @@ spec: # repo rather than using github or the like. You'll also need to # mount it into the container, below. See # https://docs.fluxcd.io/en/latest/guides/use-private-git-host.html - - name: ssh-config - configMap: - name: flux-ssh-config + # - name: ssh-config + # configMap: + # name: flux-ssh-config # The following volume is for using a customised .kube/config, # which you will need to do if you wish to have a different @@ -131,8 +131,8 @@ spec: # Include this if you need to mount a customised known_hosts # file; you'll also need the volume declared above. - - name: ssh-config - mountPath: /root/.ssh + # - name: ssh-config + # mountPath: /root/.ssh # Include this and the volume "kubeconfig" above, and the # environment entry "KUBECONFIG" below, to override the config diff --git a/workloads/kube-system/fluentd-config.yml b/workloads/kube-system/fluentd-config.yml new file mode 100644 index 0000000..ffb1ce1 --- /dev/null +++ b/workloads/kube-system/fluentd-config.yml @@ -0,0 +1,316 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + k8s-app: fluentd-logging + name: fluentd-logging + namespace: kube-system +data: + fluent.conf: | + @include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" + @include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" + @include kubernetes.conf + @include conf.d/*.conf + + + @type elasticsearch + @id out_es + @log_level info + include_tag_key true + host "#{ENV['FLUENT_ELASTICSEARCH_HOST']}" + port "#{ENV['FLUENT_ELASTICSEARCH_PORT']}" + path "#{ENV['FLUENT_ELASTICSEARCH_PATH']}" + scheme "#{ENV['FLUENT_ELASTICSEARCH_SCHEME'] || 'http'}" + ssl_verify "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERIFY'] || 'true'}" + ssl_version "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERSION'] || 'TLSv1_2'}" + user "#{ENV['FLUENT_ELASTICSEARCH_USER'] || use_default}" + password "#{ENV['FLUENT_ELASTICSEARCH_PASSWORD'] || use_default}" + reload_connections "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_CONNECTIONS'] || 'false'}" + reconnect_on_error "#{ENV['FLUENT_ELASTICSEARCH_RECONNECT_ON_ERROR'] || 'true'}" + reload_on_failure "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_ON_FAILURE'] || 'true'}" + log_es_400_reason "#{ENV['FLUENT_ELASTICSEARCH_LOG_ES_400_REASON'] || 'false'}" + logstash_prefix "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_PREFIX'] || 'logstash'}" + logstash_dateformat "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_DATEFORMAT'] || '%Y.%m.%d'}" + logstash_format "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_FORMAT'] || 'true'}" + index_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_INDEX_NAME'] || 'logstash'}" + type_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_TYPE_NAME'] || 'fluentd'}" + include_timestamp "#{ENV['FLUENT_ELASTICSEARCH_INCLUDE_TIMESTAMP'] || 'false'}" + template_name "#{ENV['FLUENT_ELASTICSEARCH_TEMPLATE_NAME'] || use_nil}" + template_file "#{ENV['FLUENT_ELASTICSEARCH_TEMPLATE_FILE'] || use_nil}" + template_overwrite "#{ENV['FLUENT_ELASTICSEARCH_TEMPLATE_OVERWRITE'] || use_default}" + sniffer_class_name "#{ENV['FLUENT_SNIFFER_CLASS_NAME'] || 'Fluent::Plugin::ElasticsearchSimpleSniffer'}" + request_timeout "#{ENV['FLUENT_ELASTICSEARCH_REQUEST_TIMEOUT'] || '5s'}" + + flush_thread_count "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_THREAD_COUNT'] || '8'}" + flush_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_INTERVAL'] || '5s'}" + chunk_limit_size "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_CHUNK_LIMIT_SIZE'] || '2M'}" + queue_limit_length "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_QUEUE_LIMIT_LENGTH'] || '32'}" + retry_max_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_RETRY_MAX_INTERVAL'] || '30'}" + retry_forever true + + + + disable.conf: "" + + kubernetes.conf: | + + + + @type tail + @id in_tail_container_logs + path /var/log/containers/*.log + pos_file /var/log/fluentd-containers.log.pos + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + + @type regexp + expression /^(? + + + + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file /var/log/fluentd-salt.pos + tag salt + + @type regexp + expression /^(? + + + + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file /var/log/fluentd-startupscript.log.pos + tag startupscript + + @type syslog + + + + + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file /var/log/fluentd-docker.log.pos + tag docker + + @type regexp + expression /^time="(? + + + + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file /var/log/fluentd-etcd.log.pos + tag etcd + + @type none + + + + + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file /var/log/fluentd-kubelet.log.pos + tag kubelet + + @type kubernetes + + + + + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file /var/log/fluentd-kube-proxy.log.pos + tag kube-proxy + + @type kubernetes + + + + + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file /var/log/fluentd-kube-apiserver.log.pos + tag kube-apiserver + + @type kubernetes + + + + + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file /var/log/fluentd-kube-controller-manager.log.pos + tag kube-controller-manager + + @type kubernetes + + + + + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file /var/log/fluentd-kube-scheduler.log.pos + tag kube-scheduler + + @type kubernetes + + + + + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file /var/log/fluentd-rescheduler.log.pos + tag rescheduler + + @type kubernetes + + + + + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file /var/log/fluentd-glbc.log.pos + tag glbc + + @type kubernetes + + + + + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file /var/log/fluentd-cluster-autoscaler.log.pos + tag cluster-autoscaler + + @type kubernetes + + + + # Example: + # 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="" asgroups="" namespace="default" uri="/api/v1/namespaces/default/pods" + # 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" + + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file /var/log/kube-apiserver-audit.log.pos + tag kube-apiserver-audit + + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(? + + + + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV.fetch('KUBERNETES_SERVICE_PORT') + '/api'}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + + + prometheus.conf: | + # Prometheus metric exposed on 0.0.0.0:24231/metrics + + @type prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" + + + + @type prometheus_output_monitor + + + systemd.conf: | + # Logs from systemd-journal for interesting services. + + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + + + fields_strip_underscores true + + read_from_head true + tag kubelet + + + # Logs from docker-systemd + + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + + + fields_strip_underscores true + + read_from_head true + tag docker.systemd + + + # Logs from systemd-journal for interesting services. + + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + + + fields_strip_underscores true + + read_from_head true + tag bootkube + diff --git a/workloads/kube-system/fluentd-daemonset-elasticsearch-rbac.yaml b/workloads/kube-system/fluentd-daemonset-elasticsearch-rbac.yaml new file mode 100644 index 0000000..6650efb --- /dev/null +++ b/workloads/kube-system/fluentd-daemonset-elasticsearch-rbac.yaml @@ -0,0 +1,118 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fluentd + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: fluentd + namespace: kube-system +rules: +- apiGroups: + - "" + resources: + - pods + - namespaces + verbs: + - get + - list + - watch + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: fluentd +roleRef: + kind: ClusterRole + name: fluentd + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: fluentd + namespace: kube-system +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: fluentd + namespace: kube-system + labels: + k8s-app: fluentd-logging + version: v1 +spec: + selector: + matchLabels: + k8s-app: fluentd-logging + version: v1 + template: + metadata: + labels: + k8s-app: fluentd-logging + version: v1 + spec: + serviceAccount: fluentd + serviceAccountName: fluentd + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: fluentd + image: fluent/fluentd-kubernetes-daemonset:v1-debian-elasticsearch + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: "es01.home.shoup.io" + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" + - name: FLUENT_ELASTICSEARCH_SCHEME + value: "http" + # Option to configure elasticsearch plugin with self signed certs + # ================================================================ + # - name: FLUENT_ELASTICSEARCH_SSL_VERIFY + # value: "true" + # Option to configure elasticsearch plugin with tls + # ================================================================ + # - name: FLUENT_ELASTICSEARCH_SSL_VERSION + # value: "TLSv1_2" + # X-Pack Authentication + # ===================== + # - name: FLUENT_ELASTICSEARCH_USER + # value: "elastic" + # - name: FLUENT_ELASTICSEARCH_PASSWORD + # value: "changeme" + + # Disable SystemD input + - name: FLUENTD_SYSTEMD_CONF + value: "disable" + # Don't parse own logs + - name: FLUENT_CONTAINER_TAIL_EXCLUDE_PATH + value: '["/var/log/containers/fluentd*"]' + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 200Mi + volumeMounts: + - name: varlog + mountPath: /var/log + - name: varlogcontainers + mountPath: /var/log/containers + readOnly: true + - name: config-volume + mountPath: /fluentd/etc + terminationGracePeriodSeconds: 30 + volumes: + - name: varlog + hostPath: + path: /var/log + - name: varlogcontainers + hostPath: + path: /var/log/containers + - name: config-volume + configMap: + name: fluentd-logging