From 74431b93ee2f800bdd1fc3586abc5b7a807d7a04 Mon Sep 17 00:00:00 2001 From: chenqz1987 Date: Wed, 17 Oct 2018 14:56:01 +0800 Subject: [PATCH] fix issue #150 and update fluentd version to 1.2.6 --- Dockerfile.fluentd | 5 +++-- assets/fluentd/plugins/out_aliyun_sls.rb | 14 +++++--------- docs/filebeat/docs.md | 2 ++ docs/fluentd/docs.md | 1 + docs/fluentd/output/aliyun_sls.md | 2 ++ pilot/fluentd_piloter.go | 13 +++++++++++-- pilot/pilot.go | 2 +- 7 files changed, 25 insertions(+), 14 deletions(-) diff --git a/Dockerfile.fluentd b/Dockerfile.fluentd index 269c917c..93de0965 100644 --- a/Dockerfile.fluentd +++ b/Dockerfile.fluentd @@ -17,8 +17,8 @@ RUN apk update && \ apk add python && \ apk add lsof && \ apk add ca-certificates wget && \ - gem install fluentd -v 1.1.0 --no-ri --no-rdoc && \ - gem install fluent-plugin-elasticsearch --no-ri --no-rdoc && \ + gem install fluentd -v 1.2.6 --no-ri --no-rdoc && \ + gem install fluent-plugin-elasticsearch -v ">=2.0.0" --no-ri --no-rdoc && \ gem install gelf -v "~> 3.0.0" --no-ri --no-rdoc && \ gem install aliyun_sls_sdk -v ">=0.0.9" --no-ri --no-rdoc && \ gem install remote_syslog_logger -v ">=1.0.1" --no-ri --no-rdoc && \ @@ -42,4 +42,5 @@ HEALTHCHECK CMD /pilot/healthz VOLUME /etc/fluentd/conf.d VOLUME /pilot/pos WORKDIR /pilot/ +ENV PILOT_TYPE=fluentd ENTRYPOINT ["/pilot/entrypoint"] diff --git a/assets/fluentd/plugins/out_aliyun_sls.rb b/assets/fluentd/plugins/out_aliyun_sls.rb index d54f6b42..8563d5a3 100644 --- a/assets/fluentd/plugins/out_aliyun_sls.rb +++ b/assets/fluentd/plugins/out_aliyun_sls.rb @@ -33,11 +33,7 @@ def shutdown end def format(tag, time, record) - if record["@target"] - [tag, time, record].to_msgpack - else - super - end + [tag, time, record].to_msgpack end def client @@ -75,15 +71,15 @@ def getLogItem(record) def write(chunk) log_list_hash = {} chunk.msgpack_each do |tag, time, record| - if record and record["@target"] - logStoreName = record["@target"] - record.delete("@target") + if record and record["_target"] + logStoreName = record["_target"] + record.delete("_target") if not log_list_hash[logStoreName] log_list_hash[logStoreName] = [] end log_list_hash[logStoreName] << getLogItem(record) else - log.warn "no @target key in record: #{record}, tag: #{tag}, time: #{time}" + log.warn "no _target key in record: #{record}, tag: #{tag}, time: #{time}" end end diff --git a/docs/filebeat/docs.md b/docs/filebeat/docs.md index 5f27bc6f..7f85d4de 100644 --- a/docs/filebeat/docs.md +++ b/docs/filebeat/docs.md @@ -17,6 +17,7 @@ docker run --rm -it \ -v /var/run/docker.sock:/var/run/docker.sock \ -v /etc/localtime:/etc/localtime \ -v /:/host:ro \ + --cap-add SYS_ADMIN \ registry.cn-hangzhou.aliyuncs.com/acs/log-pilot:0.9.5-filebeat ``` @@ -31,6 +32,7 @@ docker run --rm -it \ -v /var/run/docker.sock:/var/run/docker.sock \ -v /etc/localtime:/etc/localtime \ -v /:/host:ro \ + --cap-add SYS_ADMIN \ -e LOGGING_OUTPUT=elasticsearch \ -e ELASTICSEARCH_HOST=${ELASTICSEARCH_HOST} \ -e ELASTICSEARCH_PORT=${ELASTICSEARCH_PORT} \ diff --git a/docs/fluentd/docs.md b/docs/fluentd/docs.md index c68c5ae6..28913765 100644 --- a/docs/fluentd/docs.md +++ b/docs/fluentd/docs.md @@ -31,6 +31,7 @@ The command below run pilot with elastichsearch output, this makes log-pilot sen docker run --rm -it \ -v /var/run/docker.sock:/var/run/docker.sock \ -v /:/host:ro \ + --cap-add SYS_ADMIN \ -e LOGGING_OUTPUT=elasticsearch \ -e ELASTICSEARCH_HOST=${ELASTICSEARCH_HOST} \ -e ELASTICSEARCH_PORT=${ELASTICSEARCH_PORT} \ diff --git a/docs/fluentd/output/aliyun_sls.md b/docs/fluentd/output/aliyun_sls.md index b1adaacb..53fed248 100644 --- a/docs/fluentd/output/aliyun_sls.md +++ b/docs/fluentd/output/aliyun_sls.md @@ -36,11 +36,13 @@ docker run --rm -it \ -v /var/run/docker.sock:/var/run/docker.sock \ -v /etc/localtime:/etc/localtime \ -v /:/host:ro \ + --cap-add SYS_ADMIN \ -e LOGGING_OUTPUT=aliyun_sls \ -e ALIYUNSLS_PROJECT="your-aliyun-sls-project-name" \ -e ALIYUNSLS_REGION_ENDPOINT=cn-hangzhou.log.aliyuncs.com \ -e ALIYUNSLS_ACCESS_KEY_ID="your-access-key-id" \ -e ALIYUNSLS_ACCESS_KEY_SECRET="your-access-key-secret" \ + -e ALIYUNSLS_NEED_CREATE_LOGSTORE="true" \ registry.cn-hangzhou.aliyuncs.com/acs/log-pilot:0.9.5-fluentd ```` diff --git a/pilot/fluentd_piloter.go b/pilot/fluentd_piloter.go index 2a7b1798..911e8b7f 100644 --- a/pilot/fluentd_piloter.go +++ b/pilot/fluentd_piloter.go @@ -7,6 +7,8 @@ import ( "os/exec" "syscall" "time" + "strconv" + "strings" ) const ( @@ -17,6 +19,7 @@ const ( FLUENTD_PLUGINS = FLUENTD_BASE_CONF + "/plugins" ENV_FLUENTD_OUTPUT = "FLUENTD_OUTPUT" + ENV_FLUENTD_WORKER = "FLUENTD_WORKER" ) var fluentd *exec.Cmd @@ -39,9 +42,15 @@ func (p *FluentdPiloter) Start() error { } log.Info("starting fluentd") + worker := os.Getenv(ENV_FLUENTD_WORKER) + if _, err := strconv.Atoi(worker); worker == "" || err != nil { + worker = "1" + } + fluentd = exec.Command(FLUENTD_EXEC_CMD, "-c", FLUENTD_CONF_FILE, - "-p", FLUENTD_PLUGINS) + "-p", FLUENTD_PLUGINS, + "--workers", worker) fluentd.Stderr = os.Stderr fluentd.Stdout = os.Stdout err := fluentd.Start() @@ -108,7 +117,7 @@ func shell(command string) string { if err != nil { fmt.Printf("error %v", err) } - return string(out) + return strings.TrimSpace(string(out)) } func (p *FluentdPiloter) GetConfHome() string { diff --git a/pilot/pilot.go b/pilot/pilot.go index a1aa05e2..07a04962 100644 --- a/pilot/pilot.go +++ b/pilot/pilot.go @@ -646,7 +646,7 @@ func (p *Pilot) getLogConfigs(jsonLogPath string, mounts []types.MountPoint, lab var labelNames []string //sort keys - for k, _ := range labels { + for k := range labels { labelNames = append(labelNames, k) }