ci: release v1.8.7

Laisky · Laisky · commit f9a82e563c80 · 2019-05-23T11:10:06.000+08:00
diff --git a/.docker/Dockerfile b/.docker/Dockerfile
@@ -1,4 +1,4 @@
-FROM registry:5000/gobase:1.12.1-alpine3.9 AS gobuild
+FROM registry:5000/gobase:1.12.5-alpine3.9 AS gobuild
 
 ENV GO111MODULE=on
 WORKDIR /go-fluentd
diff --git a/.docker/README.md b/.docker/README.md
@@ -17,8 +17,8 @@ go mod download
 go mod vendor
 
 # build base image
-docker build . -f ./.docker/gobase.Dockerfile -t registry:5000/gobase:1.12.1-alpine3.9
-docker push registry:5000/gobase:1.12.1-alpine3.9
+docker build . -f ./.docker/gobase.Dockerfile -t registry:5000/gobase:1.12.5-alpine3.9
+docker push registry:5000/gobase:1.12.5-alpine3.9
 
 # build image
 docker build . -f ./.docker/Dockerfile -t registry:5000/go-fluentd:1.8.2
@@ -38,8 +38,8 @@ build on machine that should installed docker.
 
 ```sh
 # build golang-stretch
-docker build . -f ./.docker/golang-stretch.Dockerfile -t registry:5000/golang:1.12.1-stretch
-docker push registry:5000/golang:1.12.1-stretch
+docker build . -f ./.docker/golang-stretch.Dockerfile -t registry:5000/golang:1.12.5-stretch
+docker push registry:5000/golang:1.12.5-stretch
 
 # build mfs-stretch
 docker build . -f ./.docker/mfs-stretch.Dockerfile -t registry:5000/mfs-stretch:20190116
diff --git a/.docker/forward.Dockerfile b/.docker/forward.Dockerfile
@@ -2,7 +2,7 @@
 # docker run -it --rm --cap-add SYS_ADMIN --device /dev/fuse -e MFS_MASTER=mfs-master.sit.ptcloud.t.home -e TZ=Asia/Shanghai -v /opt/configs/go-fluentd/forward:/forward registry:5000/go-fluentd-forward:666 /bin/sh
 # sh startApp.sh
 # cp /forward/settings.yml /data/Sit/go-fluentd/settings/.
-FROM registry:5000/golang:1.12.1-stretch AS gobin
+FROM registry:5000/golang:1.12.5-stretch AS gobin
 
 ENV GO111MODULE=on
 
diff --git a/.docker/gobase.Dockerfile b/.docker/gobase.Dockerfile
@@ -1,6 +1,6 @@
-# docker build . -f ./.docker/gobase.Dockerfile -t registry:5000/gobase:1.12.1-alpine3.9
-# docker push registry:5000/gobase:1.12.1-alpine3.9
-FROM golang:1.12.1-alpine3.9
+# docker build . -f ./.docker/gobase.Dockerfile -t registry:5000/gobase:1.12.5-alpine3.9
+# docker push registry:5000/gobase:1.12.5-alpine3.9
+FROM golang:1.12.5-alpine3.9
 
 # http proxy
 ENV HTTP_PROXY=http://172.16.4.26:17777
diff --git a/.docker/golang-stretch.Dockerfile b/.docker/golang-stretch.Dockerfile
@@ -1,6 +1,6 @@
-# docker build . -f ./.docker/golang-stretch.Dockerfile -t registry:5000/golang:1.12.1-stretch
-# docker push registry:5000/golang:1.12.1-stretch
-FROM golang:1.12.1-stretch
+# docker build . -f ./.docker/golang-stretch.Dockerfile -t registry:5000/golang:1.12.5-stretch
+# docker push registry:5000/golang:1.12.5-stretch
+FROM golang:1.12.5-stretch
 
 # http proxy
 ENV HTTP_PROXY=http://172.16.4.26:17777
diff --git a/.docker/test.Dockerfile b/.docker/test.Dockerfile
@@ -1,6 +1,6 @@
 # docker build . -f ./.docker/test.Dockerfile -t registry:5000/go-fluentd-test:v1
 # docker push registry:5000/go-fluentd-test:v1
-FROM registry:5000/gobase:1.12.1-alpine3.9
+FROM registry:5000/gobase:1.12.5-alpine3.9
 
 # http proxy
 ENV HTTP_PROXY=http://172.16.4.26:17777
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -46,7 +46,7 @@ test:run:
     - public
   image: registry:5000/docker:latest
   script:
-    - docker pull registry:5000/gobase:1.12.1-alpine3.9  # pull latest image
+    - docker pull registry:5000/gobase:1.12.5-alpine3.9  # pull latest image
     - docker build . -f ./.docker/test.Dockerfile -t go-fluentd-test:$DOCKER_TAG
     - docker run -i --rm go-fluentd-test:$DOCKER_TAG
   retry: 1
diff --git a/senders/elasticsearch.go b/senders/elasticsearch.go
@@ -71,10 +71,12 @@ func (s *ElasticSearchSender) GetName() string {
 	return s.Name
 }
 
-type BulkOpCtx struct {
-	body     []byte
+type bulkOpCtx struct {
 	buf      *bytes.Buffer
 	gzWriter *gzip.Writer
+	cnt      []byte
+	starting []byte
+	msg      *libs.FluentMsg
 }
 
 func (s *ElasticSearchSender) getMsgStarting(msg *libs.FluentMsg) ([]byte, error) {
@@ -90,32 +92,30 @@ func (s *ElasticSearchSender) getMsgStarting(msg *libs.FluentMsg) ([]byte, error
 	return []byte("{\"index\": {\"_index\": \"" + index + "\", \"_type\": \"logs\"}}\n"), nil
 }
 
-func (s *ElasticSearchSender) SendBulkMsgs(ctx *BulkOpCtx, msgs []*libs.FluentMsg) (err error) {
-	cnt := []byte{}
-	var starting []byte
-	for _, m := range msgs {
-		if starting, err = s.getMsgStarting(m); err != nil {
+func (s *ElasticSearchSender) SendBulkMsgs(ctx *bulkOpCtx, msgs []*libs.FluentMsg) (err error) {
+	ctx.cnt = ctx.cnt[:0]
+	for _, ctx.msg = range msgs {
+		if ctx.starting, err = s.getMsgStarting(ctx.msg); err != nil {
 			utils.Logger.Warn("try to generate bulk index got error", zap.Error(err))
 			continue
 		}
 
-		b, err := json.Marshal(m.Message)
+		b, err := json.Marshal(ctx.msg.Message)
 		if err != nil {
 			return errors.Wrap(err, "try to marshal messages got error")
 		}
 
 		utils.Logger.Debug("prepare bulk content send to es",
-			zap.ByteString("starting", starting),
+			zap.ByteString("starting", ctx.starting),
 			zap.ByteString("body", b))
-		cnt = append(cnt, starting...)
-		cnt = append(cnt, b...)
-		cnt = append(cnt, '\n')
+		ctx.cnt = append(ctx.cnt, ctx.starting...)
+		ctx.cnt = append(ctx.cnt, b...)
+		ctx.cnt = append(ctx.cnt, '\n')
 	}
 
 	ctx.buf.Reset()
 	ctx.gzWriter.Reset(ctx.buf)
-	// ctx.gzWriter = gzip.NewWriter(ctx.buf)
-	if _, err = ctx.gzWriter.Write(cnt); err != nil {
+	if _, err = ctx.gzWriter.Write(ctx.cnt); err != nil {
 		return errors.Wrap(err, "try to compress messages got error")
 	}
 
@@ -205,8 +205,10 @@ func (s *ElasticSearchSender) Spawn(tag string) chan<- *libs.FluentMsg {
 				iBatch           = 0
 				lastT            = time.Unix(0, 0)
 				err              error
-				ctx              = &BulkOpCtx{}
-				nRetry, j        int
+				ctx              = &bulkOpCtx{
+					cnt: []byte{},
+				}
+				nRetry, j int
 			)
 
 			ctx.buf = &bytes.Buffer{}
@@ -244,12 +246,13 @@ func (s *ElasticSearchSender) Spawn(tag string) chan<- *libs.FluentMsg {
 				if err = s.SendBulkMsgs(ctx, msgBatchDelivery); err != nil {
 					nRetry++
 					if nRetry > maxRetry {
-						utils.Logger.Error("try send message got error", zap.Error(err), zap.String("tag", tag))
-						utils.Logger.Error("discard msg since of sender err", zap.String("tag", msg.Tag), zap.Int("num", len(msgBatchDelivery)))
+						utils.Logger.Error("try send message got error",
+							zap.Error(err),
+							zap.String("tag", tag),
+							zap.Int("num", len(msgBatchDelivery)))
 						for _, msg = range msgBatchDelivery {
 							s.discardWithoutCommitChan <- msg
 						}
-
 						continue
 					}
 					goto SEND_MSG
diff --git a/senders/httpforward.go b/senders/httpforward.go
@@ -73,7 +73,7 @@ func (s *HTTPSender) Spawn(tag string) chan<- *libs.FluentMsg {
 				msgBatchDelivery []*libs.FluentMsg
 				iBatch           = 0
 				lastT            = time.Unix(0, 0)
-				ctx              = &BulkOpCtx{}
+				ctx              = &bulkOpCtx{}
 				err              error
 			)
 
@@ -136,7 +136,7 @@ func (s *HTTPSender) Spawn(tag string) chan<- *libs.FluentMsg {
 	return inChan
 }
 
-func (s *HTTPSender) SendBulkMsgs(ctx *BulkOpCtx, msgs []*libs.FluentMsg) (err error) {
+func (s *HTTPSender) SendBulkMsgs(ctx *bulkOpCtx, msgs []*libs.FluentMsg) (err error) {
 	msgCnts := make([]map[string]interface{}, len(msgs))
 	for i, m := range msgs {
 		msgCnts[i] = m.Message

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-FROM registry:5000/gobase:1.12.1-alpine3.9 AS gobuild`
	`1`	`+FROM registry:5000/gobase:1.12.5-alpine3.9 AS gobuild`
`2`	`2`
`3`	`3`	`ENV GO111MODULE=on`
`4`	`4`	`WORKDIR /go-fluentd`