基于kubernetes的日志监控系统安装 -欧洲杯足彩官网

1.场景

我们在生产环境中需要对系统的各种日志进行采集、查询和分析。本例演示使用fluentd进行日志采集，elasticsearch进行日志存储，kibana进行日志查询分析。

2.安装

2.1 创建dashboard用户

sa.yml:

apiversion: v1
kind: serviceaccount
metadata:
  name: dashboard
  namespace: kube-system
---
apiversion: rbac.authorization.k8s.io/v1beta1
kind: clusterrolebinding
metadata:
  name: dashboard
roleref:
  kind: clusterrole
  name: view
  apigroup: rbac.authorization.k8s.io
subjects:
  - kind: serviceaccount
    name: dashboard
    namespace: kube-system

2.2 创建persistentvolume

创建persistentvolume用作elasticsearch存储所用的磁盘:

pv.yml:

apiversion: v1
kind: persistentvolume
metadata:
  name: elk-log-pv
spec:
  capacity:
    storage: 5gi
  accessmodes:
    - readwritemany
   nfs:
     path: /opt/data/kafka0
     server: 192.168.1.140
     readonly: false

2.3 fluentd的配置(configmap)

若无此步的话，fluentd-es镜像会有下面的错误：

2018-03-07 08:35:18  0000 [info]: adding filter pattern="kubernetes.**" type="kubernetes_metadata"
2018-03-07 08:35:19  0000 [error]: config error file="/etc/td-agent/td-agent.conf" error="invalid kubernetes api v1 endpoint https://172.21.0.1:443/api: ssl_connect returned=1 errno=0 state=error: certificate verify failed"
2018-03-07 08:35:19  0000 [info]: process finished code=256

所以需要配置configmap：

cm.yml:

apiversion: v1
kind: configmap
metadata:
  name: fluentd-conf
  namespace: kube-system
data:
  td-agent.conf: |
    
      type null
    
    # example:
    # {"log":"[info:2016-02-16t16:04:05.930-08:00] some log text here\n","stream":"stdout","time":"2016-02-17t00:04:05.931087621z"}
    
      type tail
      path /var/log/containers/*.log
      pos_file /var/log/es-containers.log.pos
      time_format %y-%m-%dt%h:%m:%s.%nz
      tag kubernetes.*
      format json
      read_from_head true
    
    
      type kubernetes_metadata
      verify_ssl false
    
    
      type tail
      format syslog
      path /var/log/messages
      pos_file /var/log/messages.pos
      tag system
    
    
       type elasticsearch
       user "#{env['fluent_elasticsearch_user']}"
       password "#{env['fluent_elasticsearch_password']}"
       log_level info
       include_tag_key true
       host elasticsearch-logging
       port 9200
       logstash_format true
       # set the chunk limit the same as for fluentd-gcp.
       buffer_chunk_limit 2m
       # cap buffer memory usage to 2mib/chunk * 32 chunks = 64 mib
       buffer_queue_limit 32
       flush_interval 5s
       # never wait longer than 5 minutes between retries.
       max_retry_wait 30
       # disable the limit on the number of retries (retry forever).
       disable_retry_limit
       # use multiple threads for processing.
       num_threads 8
    

2.4 整体配置

logging.yml:

apiversion: v1
kind: replicationcontroller
metadata:
  name: elasticsearch-logging-v1
  namespace: kube-system
  labels:
    k8s-app: elasticsearch-logging
    version: v1
    kubernetes.io/cluster-service: "true"
spec:
  replicas: 2
  selector:
    k8s-app: elasticsearch-logging
    version: v1
  template:
    metadata:
      labels:
        k8s-app: elasticsearch-logging
        version: v1
        kubernetes.io/cluster-service: "true"
    spec:
      serviceaccount: dashboard
      containers:
      - image: registry.cn-hangzhou.aliyuncs.com/google-containers/elasticsearch:v2.4.1-1
        name: elasticsearch-logging
        resources:
          # need more cpu upon initialization, therefore burstable class
          limits:
            cpu: 1000m
          requests:
            cpu: 100m
        ports:
        - containerport: 9200
          name: db
          protocol: tcp
        - containerport: 9300
          name: transport
          protocol: tcp
        volumemounts:
        - name: es-persistent-storage
          mountpath: /data
        env:
        - name: "namespace"
          valuefrom:
            fieldref:
              fieldpath: metadata.namespace
      volumes:
      - name: es-persistent-storage
        persistentvolumeclaim:
          claimname: elk-log
---
kind: persistentvolumeclaim
apiversion: v1
metadata:
  name: elk-log
  namespace: kube-system
spec:
  accessmodes:
    - readwritemany
  resources:
    requests:
      storage: 5gi
  #selector:
  #  matchlabels:
  #    release: "stable"
  #  matchexpressions:
  #    - {key: environment, operator: in, values: [dev]}
---
apiversion: v1
kind: service
metadata:
  name: elasticsearch-logging
  namespace: kube-system
  labels:
    k8s-app: elasticsearch-logging
    kubernetes.io/cluster-service: "true"
    kubernetes.io/name: "elasticsearch"
spec:
  ports:
  - port: 9200
    protocol: tcp
    targetport: db
  selector:
    k8s-app: elasticsearch-logging
---
apiversion: extensions/v1beta1
kind: daemonset
metadata:
  name: fluentd-es-v1.22
  namespace: kube-system
  labels:
    k8s-app: fluentd-es
    kubernetes.io/cluster-service: "true"
    version: v1.22
spec:
  template:
    metadata:
      labels:
        k8s-app: fluentd-es
        kubernetes.io/cluster-service: "true"
        version: v1.22
      # this annotation ensures that fluentd does not get evicted if the node
      # supports critical pod annotation based priority scheme.
      # note that this does not guarantee admission on the nodes (#40573).
      annotations:
        scheduler.alpha.kubernetes.io/critical-pod: ''
        scheduler.alpha.kubernetes.io/tolerations: '[{"key": "node.alpha.kubernetes.io/ismaster", "effect": "noschedule"}]'
    spec:
      serviceaccount: dashboard
      containers:
      - name: fluentd-es
        image: registry.cn-hangzhou.aliyuncs.com/google-containers/fluentd-elasticsearch:1.22
        command:
          - '/bin/sh'
          - '-c'
          - '/usr/sbin/td-agent 2>&1 >> /var/log/fluentd.log'
        resources:
          limits:
            memory: 200mi
          requests:
            cpu: 100m
            memory: 200mi
        volumemounts:
        - name: varlog
          mountpath: /var/log
        - name: varlibdockercontainers
          mountpath: /var/lib/docker/containers
        - name: config-volume
          mountpath: /etc/td-agent/
          readonly: true
      #nodeselector:
      #  alpha.kubernetes.io/fluentd-ds-ready: "true"
      terminationgraceperiodseconds: 30
      volumes:
      - name: varlog
        hostpath:
          path: /var/log
      - name: varlibdockercontainers
        hostpath:
          path: /var/lib/docker/containers
      - name: config-volume
        configmap:
          name: fluentd-conf
---
apiversion: extensions/v1beta1
kind: deployment
metadata:
  name: kibana-logging
  namespace: kube-system
  labels:
    k8s-app: kibana-logging
    kubernetes.io/cluster-service: "true"
spec:
  replicas: 1
  selector:
    matchlabels:
      k8s-app: kibana-logging
  template:
    metadata:
      labels:
        k8s-app: kibana-logging
    spec:
      containers:
      - name: kibana-logging
        image: registry.cn-hangzhou.aliyuncs.com/google-containers/kibana:v4.6.1-1
        resources:
          # keep request = limit to keep this container in guaranteed class
          limits:
            cpu: 100m
          requests:
            cpu: 100m
        env:
          - name: "elasticsearch_url"
            value: "http://elasticsearch-logging:9200"
          - name: "kibana_base_url"
            value: "/api/v1/proxy/namespaces/kube-system/services/kibana-logging"
        ports:
        - containerport: 5601
          name: ui
          protocol: tcp
---
apiversion: v1
kind: service
metadata:
  name: kibana-logging
  namespace: kube-system
  labels:
    k8s-app: kibana-logging
    kubernetes.io/cluster-service: "true"
    kubernetes.io/name: "kibana"
spec:
  ports:
  - port: 5601
    protocol: tcp
    targetport: ui
  selector:
    k8s-app: kibana-logging
  type: clusterip

2.5 安装

kubectl apply -f sa.yml
kubectl apply -f cm.yml
kubectl apply -f pv.yml
kubectl apply -f logging.yml

2.6 验证

• 上步完成后要等待相当长的时间，请耐心等待。

• 查看kibana与elasticsearch访问地址:kubectl cluster-info

  kubernetes master is running at https://master-ip:6443
  elasticsearch is running at https://master-ip:6443/api/v1/namespaces/kube-system/services/elasticsearch-logging/proxy
  heapster is running at https://master-ip:6443/api/v1/namespaces/kube-system/services/heapster/proxy
  kibana is running at https://master-ip:6443/api/v1/namespaces/kube-system/services/kibana-logging/proxy
  kubedns is running at https://master-ip:6443/api/v1/namespaces/kube-system/services/kube-dns/proxy
  monitoring-influxdb is running at https://master-ip:6443/api/v1/namespaces/kube-system/services/monitoring-influxdb/proxy
  

• 启动客户端代理kubectl proxy,访问:http://localhost:8001/api/v1/namespaces/kube-system/services/kibana-logging/proxy

3 源码地址