Merge pull request #77 from rchicoli/development

set time zone

Author: rchicoli

Dockerfile

@@ -15,6 +15,10 @@ RUN CGO_ENABLED=0 go build -v -a -installsuffix cgo -o /usr/bin/docker-log-elast
 FROM alpine:3.7
 
 # RUN apk --no-cache add ca-certificates
+RUN apk --no-cache add tzdata
+# TZ required to set the localtime
+# TZ can be set with docker plugin command
+
 COPY --from=builder /usr/bin/docker-log-elasticsearch /usr/bin/
 WORKDIR /usr/bin
 ENTRYPOINT [ "/usr/bin/docker-log-elasticsearch" ]

README.md

@@ -31,11 +31,6 @@ This application is under active development and will continue to be modified an
 
 You need to install Docker Engine >= 1.12 and Elasticsearch application. Additional information about Docker plugins [can be found here](https://docs.docker.com/engine/extend/plugins_logging/).
 
-## Incompatible docker version
-
-It was found a bug with docker version `17.09.0~ce`. Currently I've been developing this plugin using the docker version `17.05.0~ce`.
-Before going stable I will add a cross test for multiple docker versions.
-
 ### How to install
 
 The following command will download and enable the plugin.
@@ -44,6 +39,15 @@ The following command will download and enable the plugin.
 docker plugin install rchicoli/docker-log-elasticsearch:latest --alias elasticsearch
 ```
 
+### How to configure
+
+The plugin must be disabled in order to change the [plugin settings](https://docs.docker.com/engine/reference/commandline/plugin_set/)
+
+| Environment | Description | Default Value |
+| ----- | ----------- | -------------- |
+| LOG_LEVEL | log level to output for plugin logs (debug, info, warn, error) | info |
+| TZ        | time zone to generate new indexes at midnight | Europe/Berlin |
+
 ### How to use
 
 #### Prerequisites
@@ -55,7 +59,7 @@ Before creating a docker container, a healthy instance of Elasticsearch service
 | Key | Default Value | Required |
 | --- | ------------- | -------- |
 | elasticsearch-fields | containerID,containerName,containerImageName,containerCreated | no |
-| elasticsearch-index | docker | no  |
+| elasticsearch-index | docker-%Y.%m.%d | no  |
 | elasticsearch-insecure | false | no |
 | elasticsearch-password | no | no |  |
 | elasticsearch-sniff | yes | no | |
@@ -87,17 +91,18 @@ Before creating a docker container, a healthy instance of Elasticsearch service
   - *index* to write log messages to
   - *examples*: docker, logging-%F, docker-%Y.%m.%d
 
-```bash
-# FORMAT controls the output.  Interpreted sequences are:
-%b     locale's abbreviated month name (Jan)
-%B     locale's full month name (January)
-%d     day of month (01)
-%F     full date; same as %Y.%m.%d
-%j     day of year (001..366)
-%m     month (01..12)
-%y     last two digits of year (00..99)
-%Y     year (2018)
-```
+Interpreted sequences are:
+
+| Regex | Description |
+| ----- | ----------- |
+| %b    | locale's abbreviated month name (Jan) |
+| %B    | locale's full month name (January) |
+| %d    | day of month (01) |
+| %F    | full date; same as %Y.%m.%d |
+| %j    | day of year (001..366) |
+| %m    | month (01..12) |
+| %y    | last two digits of year (00..99) |
+| %Y    | year (2018) |
 
 ###### elasticsearch-username ######
   - *username* to authenticate to a secure Elasticsearch cluster
@@ -344,8 +349,8 @@ docker run -ti --rm --log-driver rchicoli/docker-log-elasticsearch:development -
 
 There are some limitations so far, which will be improved at some point.
 
-  - grok parses everything to a string field, convertion type is not possible at the moment
-  - grok-pattern-from requires the file to be inside the plugin's rootfs
+  - grok parses everything to a string field, convertion type is only possible if a template has been added to elasticsearch
+  - grok-pattern-from requires the file to be inside the plugin's rootfs. Alternative is the plugin's mount source.
 
 ### Description of fields
 

pkg/docker/config.go

@@ -49,8 +49,7 @@ type Grok struct {
 
 func defaultLogOpt() LogOpt {
 	return LogOpt{
-		index: "docker",
-		// index:    "docker-%F",
+		index:    "docker-%Y.%m.%d",
 		tzpe:     "log",
 		timeout:  1,
 		fields:   "containerID,containerName,containerImageName,containerCreated",

pkg/docker/driver.go

@@ -171,14 +171,18 @@ func (d *Driver) StartLogging(file string, info logger.Info) error {
 	c.pipeline.inputCh = make(chan logdriver.LogEntry)
 	c.pipeline.outputCh = make(chan LogMessage)
 
-	c.cron = cron.New()
-	c.indexName = indexRegex(time.Now(), config.index)
-	c.cron.AddFunc("@daily", func() {
-		d.mu.Lock()
+	if indexFlag(config.index) {
 		c.indexName = indexRegex(time.Now(), config.index)
-		d.mu.Unlock()
-	})
-	c.cron.Start()
+		c.cron = cron.New()
+		c.cron.AddFunc("@daily", func() {
+			d.mu.Lock()
+			c.indexName = indexRegex(time.Now(), config.index)
+			d.mu.Unlock()
+		})
+		c.cron.Start()
+	} else {
+		c.indexName = config.index
+	}
 
 	c.pipeline.group.Go(func() error {
 

pkg/docker/index.go

@@ -14,6 +14,13 @@ import (
 // to replace with a defined regular expression
 var percent = regexp.MustCompile("%.")
 
+func indexFlag(indexRegex string) bool {
+	if strings.Contains(indexRegex, "%") {
+		return true
+	}
+	return false
+}
+
 func indexRegex(now time.Time, indexRegex string) string {
 
 	// %b     locale's abbreviated month name (Jan)

plugin/config.json

@@ -18,6 +18,14 @@
             "Settable": [
                 "value"
             ]
+        },
+        {
+            "Name": "TZ",
+            "Description": "Set time zone to generate new indexes at midnight",
+            "Value": "Europe/Berlin",
+            "Settable": [
+                "value"
+            ]
         }
     ],
     "Network": {

tests/acceptance-tests/01-defaults.sh

@@ -70,7 +70,7 @@ function test_default_fields_are_filled_out(){
   basht_assert "echo '${output}' | jq -r '.hits.hits[0]._source.message'"            == "$message"
   basht_assert "echo '${output}' | jq -r '.hits.hits[0]._source.source'"             == "stdout"
   basht_assert "echo '${output}' | jq -r '.hits.hits[0]._source.partial'"            == "true"
-  basht_assert "echo '${output}' | jq -r '.hits.hits[0]._source.timestamp'"          =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]+Z$
+  basht_assert "echo '${output}' | jq -r '.hits.hits[0]._source.timestamp'"          =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]+(\+02:00|Z)$
   basht_assert "echo '${output}' | jq -r '.hits.hits[0]._source[]' | wc -l"          == 8
 
 }

tests/helpers.bash

@@ -3,7 +3,7 @@
 ELASTICSEARCH_IP="172.31.0.2"
 ELASTICSEARCH_PORT="9200"
 
-ELASTICSEARCH_INDEX="docker"
+ELASTICSEARCH_INDEX="docker-$(date +%Y.%m.%d)"
 ELASTICSEARCH_TYPE="log"
 
 WEBAPPER_IP="172.31.0.3"