Skip to content

Commit db7df34

Browse files
LaiskyLaisky
committed
ci: release v1.10.4
1 parent 86d2906 commit db7df34

32 files changed

+953
-391
lines changed

‎.docker/Dockerfile

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,4 @@
1-
FROM registry:5000/gobase:1.12.7-alpine3.10 AS gobuild
2-
3-
# http proxy
4-
ENV HTTP_PROXY=http://172.16.4.26:17777
5-
ENV HTTPS_PROXY=http://172.16.4.26:17777
1+
FROM registry:5000/gobase:1.12.9-alpine3.10 AS gobuild
62

73
ENV GO111MODULE=on
84
WORKDIR /go-fluentd

‎.docker/README.md

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@ go mod download
1717
go mod vendor
1818

1919
# build base image
20-
docker build . -f ./.docker/gobase.Dockerfile -t registry:5000/gobase:1.12.7-alpine3.10
21-
docker push registry:5000/gobase:1.12.7-alpine3.10
20+
docker build . -f ./.docker/gobase.Dockerfile -t registry:5000/gobase:1.12.9-alpine3.10
21+
docker push registry:5000/gobase:1.12.9-alpine3.10
2222

2323
# build image
2424
docker build . -f ./.docker/Dockerfile -t registry:5000/go-fluentd:1.8.2
@@ -38,8 +38,8 @@ build on machine that should installed docker.
3838

3939
```sh
4040
# build golang-stretch
41-
docker build . -f ./.docker/golang-stretch.Dockerfile -t registry:5000/golang:1.12.6-stretch
42-
docker push registry:5000/golang:1.12.6-stretch
41+
docker build . -f ./.docker/golang-stretch.Dockerfile -t registry:5000/golang:1.12.9-stretch
42+
docker push registry:5000/golang:1.12.9-stretch
4343

4444
# build mfs-stretch
4545
docker build . -f ./.docker/mfs-stretch.Dockerfile -t registry:5000/mfs-stretch:20190116

‎.docker/forward.Dockerfile

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,7 @@
22
# docker run -it --rm --cap-add SYS_ADMIN --device /dev/fuse -e MFS_MASTER=mfs-master.sit.ptcloud.t.home -e TZ=Asia/Shanghai -v /opt/configs/go-fluentd/forward:/forward registry:5000/go-fluentd-forward:666 /bin/sh
33
# sh startApp.sh
44
# cp /forward/settings.yml /data/Sit/go-fluentd/settings/.
5-
FROM registry:5000/golang:1.12.6-stretch AS gobin
6-
7-
# http proxy
8-
ENV HTTP_PROXY=http://172.16.4.26:17777
9-
ENV HTTPS_PROXY=http://172.16.4.26:17777
5+
FROM registry:5000/golang:1.12.9-stretch AS gobin
106

117
ENV GO111MODULE=on
128
WORKDIR /go-fluentd

‎.docker/gobase.Dockerfile

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,6 @@
1-
# docker build . -f ./.docker/gobase.Dockerfile -t registry:5000/gobase:1.12.7-alpine3.10
2-
# docker push registry:5000/gobase:1.12.7-alpine3.10
3-
FROM golang:1.12.7-alpine3.10
4-
5-
# http proxy
6-
ENV HTTP_PROXY=http://172.16.4.26:17777
7-
ENV HTTPS_PROXY=http://172.16.4.26:17777
1+
# docker build . -f ./.docker/gobase.Dockerfile -t registry:5000/gobase:1.12.9-alpine3.10
2+
# docker push registry:5000/gobase:1.12.9-alpine3.10
3+
FROM golang:1.12.9-alpine3.10
84

95
# run dependencies
106
RUN apk update && apk upgrade && \

‎.docker/golang-stretch.Dockerfile

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,6 @@
1-
# docker build . -f ./.docker/golang-stretch.Dockerfile -t registry:5000/golang:1.12.6-stretch
2-
# docker push registry:5000/golang:1.12.6-stretch
3-
FROM golang:1.12.6-stretch
4-
5-
# http proxy
6-
ENV HTTP_PROXY=http://172.16.4.26:17777
7-
ENV HTTPS_PROXY=http://172.16.4.26:17777
1+
# docker build . -f ./.docker/golang-stretch.Dockerfile -t registry:5000/golang:1.12.9-stretch
2+
# docker push registry:5000/golang:1.12.9-stretch
3+
FROM golang:1.12.9-stretch
84

95
# run dependencies
106
RUN apt-get update && \

‎.docker/mfs-stretch.Dockerfile

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,6 @@
22
# docker push registry:5000/mfs-stretch:20190116
33
FROM debian:stretch
44

5-
ENV HTTP_PROXY=http://172.16.4.26:17777
6-
ENV HTTPS_PROXY=http://172.16.4.26:17777
7-
85
# mfs
96
RUN apt-get update && \
107
apt-get install -y --no-install-recommends wget lsb-release fuse libfuse2 net-tools gnupg2

‎.docker/test.Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# docker build . -f ./.docker/test.Dockerfile -t registry:5000/go-fluentd-test:v1
22
# docker push registry:5000/go-fluentd-test:v1
3-
FROM registry:5000/gobase:1.12.7-alpine3.10
3+
FROM registry:5000/gobase:1.12.9-alpine3.10
44
ENV GO111MODULE=on
55

66
WORKDIR /go-fluentd

‎.gitlab-ci.yml

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ cache:
2323

2424
stages:
2525
- build
26-
- test
26+
# - test
2727
- deploy
2828

2929

@@ -41,15 +41,15 @@ build:app:
4141
retry: 1
4242

4343

44-
test:run:
45-
stage: test
46-
tags:
47-
- public
48-
image: registry:5000/docker:latest
49-
script:
50-
- docker pull registry:5000/gobase:1.12.7-alpine3.10 # pull latest image
51-
- docker build . -f ./.docker/test.Dockerfile -t go-fluentd-test:$DOCKER_TAG
52-
retry: 1
44+
# test:run:
45+
# stage: test
46+
# tags:
47+
# - public
48+
# image: registry:5000/docker:latest
49+
# script:
50+
# - docker pull registry:5000/gobase:1.12.9-alpine3.10 # pull latest image
51+
# - docker build . -f ./.docker/test.Dockerfile -t go-fluentd-test:$DOCKER_TAG
52+
# retry: 1
5353

5454

5555
marathon-deploy:

‎CHANGELOG.md

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,42 @@
22
*CURRENT*
33
---
44

5+
- 2019-08-28 (Laisky) perf: periodic gc
6+
7+
*v1.10.4*
8+
---
9+
10+
- 2019-08-28 (Laisky) perf: disable jj gc
11+
12+
*v1.10.3*
13+
---
14+
15+
- 2019-08-28 (Laisky) docs: update changelog
16+
- 2019-08-28 (Laisky) ci: upgrade go-utils
17+
- 2019-08-27 (Laisky) fix(paas-403): journal roll bug
18+
- 2019-08-27 (Laisky) build: upgrade go-utils
19+
- 2019-08-27 (Laisky) build: `http_proxy` should in lowercase
20+
- 2019-08-27 (Laisky) fix: upgrade to go v1.12.9
21+
- 2019-08-27 (Laisky) fix: upgrade to go v1.12.9
22+
- 2019-08-27 (Laisky) perf: use `NewMonotonicCounterFromN`
23+
- 2019-08-26 (Laisky) feat(paas-398): split journal into different directory by tag
24+
- 2019-08-23 (Laisky) fix(paas-397): msg disorder after acceptorPipeline, then cause concator error
25+
26+
*v1.10.2*
27+
---
28+
29+
- 2019-08-21 (Laisky) build: upgrade go-utils to v1.6.2
30+
- 2019-08-21 (Laisky) fix: format warn
31+
- 2019-08-21 (Laisky) fix(paas-397): missing content
32+
33+
*v1.10.1*
34+
---
35+
36+
- 2019-08-20 (Laisky) fix: ignore es conflict error
37+
38+
*v1.10.0*
39+
---
40+
541
- 2019-08-20 (Laisky) ci: upgrade go-utils
642
- 2019-08-20 (Laisky) ci: upgrade go-utils
743
- 2019-08-19 (Laisky) docs: update quickstart

‎acceptor.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ func (a *Acceptor) Run() {
5353
utils.Logger.Panic("try to process legacy messages got error", zap.Error(err))
5454
}
5555

56-
couter, err := utils.NewRotateCounterFromN((maxID+1)%a.MaxRotateID, a.MaxRotateID)
56+
couter, err := utils.NewMonotonicCounterFromN((maxID+1)%a.MaxRotateID, a.MaxRotateID)
5757
if err != nil {
5858
panic(fmt.Errorf("try to create counter got error: %+v", err))
5959
}

‎acceptorFilters/pipeline.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package acceptorFilters
33
import (
44
"fmt"
55
"sync"
6+
"time"
67

78
"github.com/Laisky/go-fluentd/libs"
89
"github.com/Laisky/go-fluentd/monitor"
@@ -25,6 +26,12 @@ type AcceptorPipeline struct {
2526
throttle *utils.Throttle
2627
}
2728

29+
// PendingMsg is the message wait tobe concatenate
30+
type PendingMsg struct {
31+
msg *libs.FluentMsg
32+
lastT time.Time
33+
}
34+
2835
func NewAcceptorPipeline(cfg *AcceptorPipelineCfg, filters ...AcceptorFilterItf) *AcceptorPipeline {
2936
utils.Logger.Info("NewAcceptorPipeline")
3037

‎controllor.go

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ func (c *Controllor) initJournal() *Journal {
4646
BufSizeBytes: utils.Settings.GetInt64("settings.journal.buf_file_bytes"),
4747
JournalOutChanLen: utils.Settings.GetInt("settings.journal.journal_out_chan_len"),
4848
CommitIdChanLen: utils.Settings.GetInt("settings.journal.commit_id_chan_len"),
49+
IsCompress: utils.Settings.GetBool("settings.journal.is_compress"),
4950
})
5051
}
5152

@@ -75,6 +76,8 @@ func (c *Controllor) initRecvs(env string) []recvs.AcceptorRecvItf {
7576
Addr: utils.Settings.GetString("settings.acceptor.recvs.plugins." + name + ".addr"),
7677
TagKey: utils.Settings.GetString("settings.acceptor.recvs.plugins." + name + ".tag_key"),
7778
IsRewriteTagFromTagKey: utils.Settings.GetBool("settings.acceptor.recvs.plugins." + name + ".is_rewrite_tag_from_tag_key"),
79+
ConcatMaxLen: utils.Settings.GetInt("settings.acceptor.recvs.plugins." + name + ".concat_max_len"),
80+
ConcatCfg: libs.LoadTagsMapAppendEnv(env, utils.Settings.GetStringMap("settings.acceptor.recvs.plugins."+name+".concat")),
7881
}))
7982
case "rsyslog":
8083
receivers = append(receivers, recvs.NewRsyslogRecv(&recvs.RsyslogCfg{
@@ -220,8 +223,9 @@ func (c *Controllor) initAcceptorPipeline(env string) *acceptorFilters.AcceptorP
220223
)
221224
}
222225

223-
func (c *Controllor) initTagPipeline(env string, waitCommitChan chan<- int64) *tagFilters.TagPipeline {
226+
func (c *Controllor) initTagPipeline(env string, waitCommitChan chan<- *libs.FluentMsg) *tagFilters.TagPipeline {
224227
fs := []tagFilters.TagFilterFactoryItf{}
228+
isEnableConcator := false
225229

226230
switch utils.Settings.Get("settings.tag_filters.plugins").(type) {
227231
case map[string]interface{}:
@@ -249,6 +253,7 @@ func (c *Controllor) initTagPipeline(env string, waitCommitChan chan<- int64) *t
249253
AppendTimeZone: utils.Settings.GetString("settings.tag_filters.plugins." + name + ".append_time_zone." + env),
250254
}))
251255
case "concator":
256+
isEnableConcator = true
252257
default:
253258
utils.Logger.Panic("unknown tagfilter type",
254259
zap.String("recv_type", utils.Settings.GetString("settings.tag_filters.recvs.plugins."+name+".type")),
@@ -263,13 +268,16 @@ func (c *Controllor) initTagPipeline(env string, waitCommitChan chan<- int64) *t
263268
utils.Logger.Panic("tagfilter configuration error")
264269
}
265270

271+
// PAAS-397: put concat in fluentd-recvs
266272
// concatorFilter must in the front
267-
fs = append([]tagFilters.TagFilterFactoryItf{tagFilters.NewConcatorFact(&tagFilters.ConcatorFactCfg{
268-
NFork: utils.Settings.GetInt("settings.tag_filters.plugins.concator.config.nfork"),
269-
LBKey: utils.Settings.GetString("settings.tag_filters.plugins.concator.config.lb_key"),
270-
MaxLen: utils.Settings.GetInt("settings.tag_filters.plugins.concator.config.max_length"),
271-
Plugins: tagFilters.LoadConcatorTagConfigs(env, utils.Settings.Get("settings.tag_filters.plugins.concator.plugins").(map[string]interface{})),
272-
})}, fs...)
273+
if isEnableConcator {
274+
fs = append([]tagFilters.TagFilterFactoryItf{tagFilters.NewConcatorFact(&tagFilters.ConcatorFactCfg{
275+
NFork: utils.Settings.GetInt("settings.tag_filters.plugins.concator.config.nfork"),
276+
LBKey: utils.Settings.GetString("settings.tag_filters.plugins.concator.config.lb_key"),
277+
MaxLen: utils.Settings.GetInt("settings.tag_filters.plugins.concator.config.max_length"),
278+
Plugins: tagFilters.LoadConcatorTagConfigs(env, utils.Settings.Get("settings.tag_filters.plugins.concator.plugins").(map[string]interface{})),
279+
})}, fs...)
280+
}
273281

274282
return tagFilters.NewTagPipeline(&tagFilters.TagPipelineCfg{
275283
MsgPool: c.msgPool,
@@ -292,7 +300,7 @@ func (c *Controllor) initDispatcher(waitDispatchChan chan *libs.FluentMsg, tagPi
292300
return dispatcher
293301
}
294302

295-
func (c *Controllor) initPostPipeline(env string, waitCommitChan chan<- int64) *postFilters.PostPipeline {
303+
func (c *Controllor) initPostPipeline(env string, waitCommitChan chan<- *libs.FluentMsg) *postFilters.PostPipeline {
296304
fs := []postFilters.PostFilterItf{
297305
// set the DefaultFilter as first filter
298306
postFilters.NewDefaultFilter(&postFilters.DefaultFilterCfg{
@@ -452,7 +460,7 @@ func (c *Controllor) initSenders(env string) []senders.SenderItf {
452460
return ss
453461
}
454462

455-
func (c *Controllor) initProducer(env string, waitProduceChan chan *libs.FluentMsg, commitChan chan<- int64, senders []senders.SenderItf) *Producer {
463+
func (c *Controllor) initProducer(env string, waitProduceChan chan *libs.FluentMsg, commitChan chan<- *libs.FluentMsg, senders []senders.SenderItf) *Producer {
456464
return NewProducer(
457465
&ProducerCfg{
458466
InChan: waitProduceChan,

‎dispacher_test.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,19 @@
11
package concator_test
22

33
import (
4+
"testing"
5+
46
utils "github.com/Laisky/go-utils"
57
)
68

9+
func TestFor(t *testing.T) {
10+
i := 0
11+
for i < 3 {
12+
i++
13+
}
14+
t.Log(i)
15+
}
16+
717
func init() {
818
utils.Settings.Setup("/Users/laisky/repo/pateo/configs/go-fluentd")
919
}

‎docs/README_cn.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -230,7 +230,7 @@ type TagFilterFactoryItf interface {
230230
GetName() string
231231

232232
SetMsgPool(*sync.Pool)
233-
SetCommittedChan(chan<- int64)
233+
SetCommittedChan(chan<- *libs.FluentMsg)
234234
SetDefaultIntervalChanSize(int)
235235
DiscardMsg(*libs.FluentMsg)
236236
}
@@ -272,7 +272,7 @@ Pipeline 是对每一个 msg，由外部去调用每一个 filters 的 Filter
272272
type PostFilterItf interface {
273273
SetUpstream(chan *libs.FluentMsg)
274274
SetMsgPool(*sync.Pool)
275-
SetCommittedChan(chan<- int64)
275+
SetCommittedChan(chan<- *libs.FluentMsg)
276276

277277
Filter(*libs.FluentMsg) *libs.FluentMsg
278278
DiscardMsg(*libs.FluentMsg)
@@ -301,7 +301,7 @@ type SenderItf interface {
301301
GetName() string
302302

303303
SetMsgPool(*sync.Pool)
304-
SetCommitChan(chan<- int64)
304+
SetCommitChan(chan<- *libs.FluentMsg)
305305
SetSupportedTags([]string)
306306
SetDiscardChan(chan<- *libs.FluentMsg)
307307
SetDiscardWithoutCommitChan(chan<- *libs.FluentMsg)

0 commit comments

Comments
 (0)