Merge pull request #91 from rchicoli/rewrite-bulk

reuse bulk processor, but leave custom too

Author: rchicoli

.travis.yml

@@ -58,7 +58,7 @@ matrix:
     - stage: release
       # env: MAJOR_RELEASE=true
       # it would be nice to map github labels instead
-      # env: RELEASE_TAG=0.4.0
+      env: RELEASE_TAG=0.5.0
       if: type IN (push)
       script:
         - ./scripts/git-release.sh

Makefile

@@ -9,14 +9,16 @@ SCRIPTS_DIR         ?= $(BASE_DIR)/scripts
 TESTS_DIR           ?= $(BASE_DIR)/tests
 
 CLIENT_VERSION      ?= 5
+DEBUG_LEVEL         ?= debug
+TZ                  ?= Europe/Berlin
 
 SHELL               := /bin/bash
 SYSCTL              := $(shell which sysctl)
 DOCKER_COMPOSE      := $(shell which docker-compose)
 
 .PHONY: all
 
-all: clean docker_build plugin_create plugin_enable clean
+all: clean docker_build plugin_create plugin_set plugin_enable clean
 
 local: clean build unit_tests plugin_create plugin_set plugin_enable clean
 
@@ -62,7 +64,7 @@ plugin_push:
 
 plugin_set:
 	@echo ""
-	docker plugin set $(PLUGIN_NAME):$(PLUGIN_TAG) LOG_LEVEL=debug
+	docker plugin set $(PLUGIN_NAME):$(PLUGIN_TAG) LOG_LEVEL=$(DEBUG_LEVEL) TZ=$(TZ)
 
 push: plugin_push
 

README.md

@@ -63,7 +63,7 @@ Before creating a docker container, a healthy instance of Elasticsearch service
 | elasticsearch-insecure | false | no |
 | elasticsearch-password | no | no |  |
 | elasticsearch-sniff | yes | no | |
-| elasticsearch-timeout | 1    | no  |
+| elasticsearch-timeout | 10s    | no  |
 | elasticsearch-type  | log    | no  |
 | elasticsearch-username | no | no |  |
 | elasticsearch-url   | no     | yes |
@@ -119,7 +119,7 @@ Interpreted sequences are:
 
 ###### elasticsearch-timeout ######
   - *timeout* maximum time in seconds that a connection is allowed to take
-  - *example*: 10
+  - *examples*: 300ms, 1s, 2h45m
 
 ###### elasticsearch-fields ######
   - *fields* to log to Elasticsearch Cluster
@@ -185,14 +185,13 @@ $ docker run --rm -ti \
     --log-opt elasticsearch-sniff=false \
     --log-opt elasticsearch-index=docker-%F \
     --log-opt elasticsearch-type=log \
-    --log-opt elasticsearch-timeout=10 \
+    --log-opt elasticsearch-timeout=60s \
     --log-opt elasticsearch-version=5 \
     --log-opt elasticsearch-fields=containerID,containerName,containerImageID,containerImageName,containerCreated \
     --log-opt elasticsearch-bulk-workers=1 \
     --log-opt elasticsearch-bulk-actions=1000 \
     --log-opt elasticsearch-bulk-size=1024 \
     --log-opt elasticsearch-bulk-flush-interval=1s \
-    --log-opt elasticsearch-bulk-stats=false \
         alpine echo -n "this is a test logging message"
 ```
 

pkg/docker/config.go

@@ -16,7 +16,7 @@ type Configuration struct {
 	index    string
 	tzpe     string
 	url      string
-	timeout  int
+	timeout  time.Duration
 	fields   string
 	version  string
 	username string
@@ -35,7 +35,7 @@ type Bulk struct {
 	actions       int
 	size          int
 	flushInterval time.Duration
-	stats         bool
+	// stats         bool
 }
 
 // Grok filter
@@ -51,7 +51,7 @@ func newConfiguration() Configuration {
 	return Configuration{
 		index:    "docker-%Y.%m.%d",
 		tzpe:     "log",
-		timeout:  1,
+		timeout:  10 * time.Second,
 		fields:   "containerID,containerName,containerImageName,containerCreated",
 		version:  "5",
 		sniff:    true,
@@ -60,9 +60,9 @@ func newConfiguration() Configuration {
 		Bulk: Bulk{
 			workers:       1,
 			actions:       100,
-			size:          5 << 20,
+			size:          5 << 20, // 5 MB = 0101 0000 0000 0000 0000 0000 = 5242880
 			flushInterval: 5 * time.Second,
-			stats:         false,
+			// stats:         false,
 		},
 
 		Grok: Grok{
@@ -153,7 +153,7 @@ func (c *Configuration) validateLogOpt(cfg map[string]string) error {
 				return fmt.Errorf("error: elasticsearch-version not supported: %s", v)
 			}
 		case "elasticsearch-timeout":
-			timeout, err := strconv.Atoi(v)
+			timeout, err := time.ParseDuration(v)
 			if err != nil {
 				return fmt.Errorf("error: parsing elasticsearch-timeout: %q", err)
 			}
@@ -183,12 +183,12 @@ func (c *Configuration) validateLogOpt(cfg map[string]string) error {
 				return fmt.Errorf("error: parsing elasticsearch-bulk-flush-interval: %q", err)
 			}
 			c.Bulk.flushInterval = flushInterval
-		case "elasticsearch-bulk-stats":
-			stats, err := strconv.ParseBool(v)
-			if err != nil {
-				return fmt.Errorf("error: parsing elasticsearch-bulk-stats: %q", err)
-			}
-			c.Bulk.stats = stats
+		// case "elasticsearch-bulk-stats":
+		// 	stats, err := strconv.ParseBool(v)
+		// 	if err != nil {
+		// 		return fmt.Errorf("error: parsing elasticsearch-bulk-stats: %q", err)
+		// 	}
+		// 	c.Bulk.stats = stats
 
 		case "grok-pattern":
 			c.grokPattern = v