k8s 单独部署

StatefulSet 方式部署

项目路径：/Users/liulei/service/EKS/ekstest/elk/logstash/example

StatefulSet.yaml

apiVersion: apps/v1
kind: StatefulSet
metadata:
  labels:
    app: logstash-logstash
    release: logstash
  name: logstash-logstash
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: logstash-logstash
      release: logstash
  serviceName: logstash-logstash-headless
  template:
    metadata:
      labels:
        app: logstash-logstash
        release: logstash
      name: logstash-logstash
    spec:
      containers:
      - env:
        - name: LS_JAVA_OPTS
          value: -Xmx1g -Xms1g
        # image: docker.elastic.co/logstash/logstash:7.17.5
        image: mrliulei/logstash:7.17.5
        imagePullPolicy: IfNotPresent
        name: logstash
        ports:
        - containerPort: 5044
          name: logstash
          protocol: TCP
        - containerPort: 9600
          name: monitoring
          protocol: TCP
        volumeMounts:
        - mountPath: /usr/share/logstash/pipeline
          name: logstashpipeline
      dnsPolicy: ClusterFirst
      nodeSelector:
        topology.ebs.csi.aws.com/zone: us-west-2a
      volumes:
      - configMap:
          defaultMode: 420
          name: logstash-pipeline
        name: logstashpipeline

service.yaml

apiVersion: v1
kind: Service
metadata:
  name: logstash-headless
  namespace: monitoring
spec:
  clusterIP: None
  clusterIPs:
  - None
  ports:
  - name: monitoring
    port: 9600
    protocol: TCP
    targetPort: 9600
  - name: logstash
    port: 5044
    protocol: TCP
    targetPort: 5044
  selector:
    app: logstash-logstash
  sessionAffinity: None
  type: ClusterIP
status:
  loadBalancer: {}

ConfigMap.yaml

apiVersion: v1
data:
  logstash.conf: |
    input {
        beats {
            port => 5044
        }
    }

    filter {
       if [filename] == "novel-api-log" {
            grok {
                match => {
                    # 2022-08-27 19:17:39.334 [restartedMain] INFO  [][] com.lexing.config.BaseRedisCachingConfig - cacheManager - 210 - RedisCacheManager
                        "message" => "\[%{TIMESTAMP_ISO8601:date}\]\s+\[%{GREEDYDATA:thread}\]\s+%{WORD:method}\s+\[(?<local>\w*)\]\[(?<user>\w*)\]\s+%{NOTSPACE:class}\s+-\s+%{WORD:function}\s+-\s+%{INT:line}\s+-\s+%{GREEDYDATA:content}"
                }
            }
            mutate {
                enable_metric => "false"
                remove_field => ["message", "log", "tags", "input", "agent", "host", "ecs", "@version"]
            }

            date {
                match => ["date","yyyy-MM-dd HH:mm:ss.SSS"]
                target => "date"
            }
        }
    }

    output {
        stdout {
            codec => rubydebug
        }

        if [filename] == "novel-api-log" {
            elasticsearch {
                # 每个月创建一个新的索引
                index => "novel-api_log_%{+YYYY-MM}"
                hosts => ["http://elasticsearch:9200"]
            }
        }
    }
kind: ConfigMap
metadata:
  name: logstash-pipeline
  namespace: monitoring

执行命令创建

kubectl create -f .