diff --git a/workloads/kube-system/fluentd-config.yml b/workloads/kube-system/fluentd-config.yml deleted file mode 100644 index c1420cc..0000000 --- a/workloads/kube-system/fluentd-config.yml +++ /dev/null @@ -1,363 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - labels: - k8s-app: fluentd-logging - name: fluentd-logging - namespace: kube-system -data: - fluent.conf: | - @include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" - @include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" - @include kubernetes.conf - @include nextcloud.conf - @include conf.d/*.conf - @include filters.conf - - - @type elasticsearch - @id out_es - @log_level info - include_tag_key true - host "#{ENV['FLUENT_ELASTICSEARCH_HOST']}" - port "#{ENV['FLUENT_ELASTICSEARCH_PORT']}" - path "#{ENV['FLUENT_ELASTICSEARCH_PATH']}" - scheme "#{ENV['FLUENT_ELASTICSEARCH_SCHEME'] || 'http'}" - ssl_verify "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERIFY'] || 'true'}" - ssl_version "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERSION'] || 'TLSv1_2'}" - user "#{ENV['FLUENT_ELASTICSEARCH_USER'] || use_default}" - password "#{ENV['FLUENT_ELASTICSEARCH_PASSWORD'] || use_default}" - reload_connections "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_CONNECTIONS'] || 'false'}" - reconnect_on_error "#{ENV['FLUENT_ELASTICSEARCH_RECONNECT_ON_ERROR'] || 'true'}" - reload_on_failure "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_ON_FAILURE'] || 'true'}" - log_es_400_reason "#{ENV['FLUENT_ELASTICSEARCH_LOG_ES_400_REASON'] || 'false'}" - logstash_prefix "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_PREFIX'] || 'logstash'}" - logstash_dateformat "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_DATEFORMAT'] || '%Y.%m.%d'}" - logstash_format "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_FORMAT'] || 'true'}" - index_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_INDEX_NAME'] || 'logstash'}" - type_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_TYPE_NAME'] || 'fluentd'}" - include_timestamp "#{ENV['FLUENT_ELASTICSEARCH_INCLUDE_TIMESTAMP'] || 'false'}" - template_name "#{ENV['FLUENT_ELASTICSEARCH_TEMPLATE_NAME'] || use_nil}" - template_file "#{ENV['FLUENT_ELASTICSEARCH_TEMPLATE_FILE'] || use_nil}" - template_overwrite "#{ENV['FLUENT_ELASTICSEARCH_TEMPLATE_OVERWRITE'] || use_default}" - sniffer_class_name "#{ENV['FLUENT_SNIFFER_CLASS_NAME'] || 'Fluent::Plugin::ElasticsearchSimpleSniffer'}" - request_timeout "#{ENV['FLUENT_ELASTICSEARCH_REQUEST_TIMEOUT'] || '5s'}" - - flush_thread_count "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_THREAD_COUNT'] || '8'}" - flush_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_INTERVAL'] || '5s'}" - chunk_limit_size "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_CHUNK_LIMIT_SIZE'] || '2M'}" - queue_limit_length "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_QUEUE_LIMIT_LENGTH'] || '32'}" - retry_max_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_RETRY_MAX_INTERVAL'] || '30'}" - retry_forever true - - - - disable.conf: "" - - filters.conf: | - - @type parser - key_name log - reserve_data true - - @type apache2 - - - - - @type parser - key_name log - reserve_data true - - @type regexp - expression /^(?.*) (?.*) \((?.*)\) \[(?.*)\] (?.*)$/ - - - - - @type parser - key_name log - reserve_data true - - @type json - - - - nextcloud.conf: | - - @type tail - @id in_tail_nextcloud_log - path /opt/nextcloud/nextcloud.log - pos_file /var/log/nextcloud.log.pos - tag nextcloud.log - read_from_head true - reserve_data true - - @type json - time_key time - time_format %Y-%m-%dT%H:%M:%S%z - - - - kubernetes.conf: | - - - - @type tail - @id in_tail_container_logs - path /var/log/containers/*.log - pos_file /var/log/fluentd-containers.log.pos - tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" - exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" - read_from_head true - - @type regexp - expression /^(? - - - - @type tail - @id in_tail_minion - path /var/log/salt/minion - pos_file /var/log/fluentd-salt.pos - tag salt - - @type regexp - expression /^(? - - - - @type tail - @id in_tail_startupscript - path /var/log/startupscript.log - pos_file /var/log/fluentd-startupscript.log.pos - tag startupscript - - @type syslog - - - - - @type tail - @id in_tail_docker - path /var/log/docker.log - pos_file /var/log/fluentd-docker.log.pos - tag docker - - @type regexp - expression /^time="(? - - - - @type tail - @id in_tail_etcd - path /var/log/etcd.log - pos_file /var/log/fluentd-etcd.log.pos - tag etcd - - @type none - - - - - @type tail - @id in_tail_kubelet - multiline_flush_interval 5s - path /var/log/kubelet.log - pos_file /var/log/fluentd-kubelet.log.pos - tag kubelet - - @type kubernetes - - - - - @type tail - @id in_tail_kube_proxy - multiline_flush_interval 5s - path /var/log/kube-proxy.log - pos_file /var/log/fluentd-kube-proxy.log.pos - tag kube-proxy - - @type kubernetes - - - - - @type tail - @id in_tail_kube_apiserver - multiline_flush_interval 5s - path /var/log/kube-apiserver.log - pos_file /var/log/fluentd-kube-apiserver.log.pos - tag kube-apiserver - - @type kubernetes - - - - - @type tail - @id in_tail_kube_controller_manager - multiline_flush_interval 5s - path /var/log/kube-controller-manager.log - pos_file /var/log/fluentd-kube-controller-manager.log.pos - tag kube-controller-manager - - @type kubernetes - - - - - @type tail - @id in_tail_kube_scheduler - multiline_flush_interval 5s - path /var/log/kube-scheduler.log - pos_file /var/log/fluentd-kube-scheduler.log.pos - tag kube-scheduler - - @type kubernetes - - - - - @type tail - @id in_tail_rescheduler - multiline_flush_interval 5s - path /var/log/rescheduler.log - pos_file /var/log/fluentd-rescheduler.log.pos - tag rescheduler - - @type kubernetes - - - - - @type tail - @id in_tail_glbc - multiline_flush_interval 5s - path /var/log/glbc.log - pos_file /var/log/fluentd-glbc.log.pos - tag glbc - - @type kubernetes - - - - - @type tail - @id in_tail_cluster_autoscaler - multiline_flush_interval 5s - path /var/log/cluster-autoscaler.log - pos_file /var/log/fluentd-cluster-autoscaler.log.pos - tag cluster-autoscaler - - @type kubernetes - - - - # Example: - # 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="" asgroups="" namespace="default" uri="/api/v1/namespaces/default/pods" - # 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" - - @type tail - @id in_tail_kube_apiserver_audit - multiline_flush_interval 5s - path /var/log/kubernetes/kube-apiserver-audit.log - pos_file /var/log/kube-apiserver-audit.log.pos - tag kube-apiserver-audit - - @type multiline - format_firstline /^\S+\s+AUDIT:/ - # Fields must be explicitly captured by name to be parsed into the record. - # Fields may not always be present, and order may change, so this just looks - # for a list of key="\"quoted\" value" pairs separated by spaces. - # Unknown fields are ignored. - # Note: We can't separate query/response lines as format1/format2 because - # they don't always come one after the other for a given query. - format1 /^(? - - - - @type kubernetes_metadata - @id filter_kube_metadata - kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV.fetch('KUBERNETES_SERVICE_PORT') + '/api'}" - verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" - ca_file "#{ENV['KUBERNETES_CA_FILE']}" - - - prometheus.conf: | - # Prometheus metric exposed on 0.0.0.0:24231/metrics - - @type prometheus - bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" - port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" - metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" - - - - @type prometheus_output_monitor - - - systemd.conf: | - # Logs from systemd-journal for interesting services. - - @type systemd - @id in_systemd_kubelet - matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] - - @type local - persistent true - path /var/log/fluentd-journald-kubelet-cursor.json - - - fields_strip_underscores true - - read_from_head true - tag kubelet - - - # Logs from docker-systemd - - @type systemd - @id in_systemd_docker - matches [{ "_SYSTEMD_UNIT": "docker.service" }] - - @type local - persistent true - path /var/log/fluentd-journald-docker-cursor.json - - - fields_strip_underscores true - - read_from_head true - tag docker.systemd - - - # Logs from systemd-journal for interesting services. - - @type systemd - @id in_systemd_bootkube - matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] - - @type local - persistent true - path /var/log/fluentd-journald-bootkube-cursor.json - - - fields_strip_underscores true - - read_from_head true - tag bootkube - diff --git a/workloads/kube-system/fluentd-daemonset-elasticsearch-rbac.yaml b/workloads/kube-system/fluentd-daemonset-elasticsearch-rbac.yaml deleted file mode 100644 index 8022c80..0000000 --- a/workloads/kube-system/fluentd-daemonset-elasticsearch-rbac.yaml +++ /dev/null @@ -1,125 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: fluentd - namespace: kube-system - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: fluentd - namespace: kube-system -rules: -- apiGroups: - - "" - resources: - - pods - - namespaces - verbs: - - get - - list - - watch - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: fluentd -roleRef: - kind: ClusterRole - name: fluentd - apiGroup: rbac.authorization.k8s.io -subjects: -- kind: ServiceAccount - name: fluentd - namespace: kube-system ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: fluentd - namespace: kube-system - labels: - k8s-app: fluentd-logging - version: v1 -spec: - selector: - matchLabels: - k8s-app: fluentd-logging - version: v1 - template: - metadata: - labels: - k8s-app: fluentd-logging - version: v1 - spec: - serviceAccount: fluentd - serviceAccountName: fluentd - tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule - containers: - - name: fluentd - image: fluent/fluentd-kubernetes-daemonset:v1-debian-elasticsearch - env: - - name: FLUENT_ELASTICSEARCH_HOST - value: "es01.home.shoup.io" - - name: FLUENT_ELASTICSEARCH_PORT - value: "9200" - - name: FLUENT_ELASTICSEARCH_SCHEME - value: "http" - # Option to configure elasticsearch plugin with self signed certs - # ================================================================ - # - name: FLUENT_ELASTICSEARCH_SSL_VERIFY - # value: "true" - # Option to configure elasticsearch plugin with tls - # ================================================================ - # - name: FLUENT_ELASTICSEARCH_SSL_VERSION - # value: "TLSv1_2" - # X-Pack Authentication - # ===================== - # - name: FLUENT_ELASTICSEARCH_USER - # value: "elastic" - # - name: FLUENT_ELASTICSEARCH_PASSWORD - # value: "changeme" - - # Disable SystemD input - - name: FLUENTD_SYSTEMD_CONF - value: "disable" - # Don't parse own logs - - name: FLUENT_CONTAINER_TAIL_EXCLUDE_PATH - value: '["/var/log/containers/fluentd*"]' - resources: - limits: - memory: 200Mi - requests: - cpu: 100m - memory: 200Mi - volumeMounts: - - name: varlog - mountPath: /var/log - - name: varlogcontainers - mountPath: /var/log/containers - readOnly: true - - name: config-volume - mountPath: /fluentd/etc - - name: nextcloud-data - mountPath: /opt/nextcloud/nextcloud.log - subPath: data/nextcloud.log - terminationGracePeriodSeconds: 30 - volumes: - - name: varlog - hostPath: - path: /var/log - - name: varlogcontainers - hostPath: - path: /var/log/containers - - name: config-volume - configMap: - name: fluentd-logging - - name: nextcloud-data - hostPath: - path: /opt/nextcloud/pod - type: Directory