feat: ✨ Add option to unescape strings inside log string

Author: adrienyhuel

README.md

@@ -33,6 +33,7 @@ Apache Common Log Format. The body of the block accepts the custom `placeholder`
 log {
 	format transform [<template>] {
 		placeholder <string>
+		unescape_strings <bool>
 		# other fields accepted by JSON encoder
 	}
 }

caddyfile.go

@@ -17,6 +17,8 @@ package transformencoder
 import (
 	"strings"
 
+	"strconv"
+
 	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
 )
 
@@ -30,34 +32,45 @@ import (
 // See the godoc on the LogEncoderConfig type for the syntax of
 // subdirectives that are common to most/all encoders.
 func (se *TransformEncoder) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
-	foundTemplate := 0
-outerloop:
 	for d.Next() {
 		args := d.RemainingArgs()
 		switch len(args) {
 		case 0:
 			se.Template = commonLogFormat
 		default:
-			foundTemplate = len(args)
 			se.Template = strings.Join(args, " ")
 		}
 
 		for nesting := d.Nesting(); d.NextBlock(nesting); {
-			if d.Val() == "placeholder" {
+			switch d.Val() {
+			case "placeholder":
 				d.AllArgs(&se.Placeholder)
 				// delete the `placeholder` token and the value, and reset the cursor
 				d.Delete()
 				d.Delete()
-				break outerloop
+			case "unescape_strings":
+				// require an argument
+				if !d.NextArg() {
+					return d.ArgErr()
+				}
+				b, err := strconv.ParseBool(d.Val())
+				if nil == err {
+					se.UnescapeStrings = b
+				}
+				if d.NextArg() {
+					return d.ArgErr()
+				}
+				d.Delete()
+				d.Delete()
+			default:
+				d.RemainingArgs() //consume line without getting values
 			}
 		}
 	}
 
 	d.Reset()
 	// consume the directive and the template
-	d.Next()
-	for ; foundTemplate > 0; foundTemplate-- {
-		d.Next()
-	}
+	d.RemainingArgs()
+
 	return (&se.LogEncoderConfig).UnmarshalCaddyfile(d)
 }

caddyfile_test.go

@@ -29,6 +29,7 @@ func TestUnmarshalCaddyfile(t *testing.T) {
 		Encoder          zapcore.Encoder
 		Template         string
 		Placeholder      string
+		UnescapeStrings  bool
 	}
 	type args struct {
 		d *caddyfile.Dispenser
@@ -295,17 +296,64 @@ func TestUnmarshalCaddyfile(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "transform: not template but given unescape_strings",
+			fields: fields{
+				Template:        commonLogFormat,
+				UnescapeStrings: true,
+			},
+			args: args{
+				d: caddyfile.NewTestDispenser(`transform {
+					unescape_strings true
+				}`),
+			},
+			wantErr: false,
+		},
+		{
+			name: "transform: given template and given unescape_strings",
+			fields: fields{
+				Template:        "{obj1>obj2>[0]}",
+				UnescapeStrings: true,
+			},
+			args: args{
+				d: caddyfile.NewTestDispenser(`transform "{obj1>obj2>[0]}" {
+					unescape_strings true
+				}`),
+			},
+			wantErr: false,
+		},
+		{
+			name: "transform: `placeholder` `unescape_strings` and  property sitting between other properties",
+			fields: fields{
+				Template:        "{obj1>obj2>[0]}",
+				Placeholder:     "-",
+				UnescapeStrings: true,
+				LogEncoderConfig: logging.LogEncoderConfig{
+					TimeLocal:  true,
+					TimeFormat: "iso8601",
+				},
+			},
+			args: args{
+				d: caddyfile.NewTestDispenser(`transform "{obj1>obj2>[0]}" {
+					time_local
+					placeholder -
+					unescape_strings true
+					time_format iso8601
+				}`),
+			},
+			wantErr: false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			se := &TransformEncoder{
 				Encoder: new(logging.JSONEncoder),
 			}
 			if err := se.UnmarshalCaddyfile(tt.args.d); (err != nil) != tt.wantErr {
-				t.Errorf("TransformEncoder.UnmarshalCaddyfile() error = %v, wantErr %v", err, tt.wantErr)
+				t.Fatalf("TransformEncoder.UnmarshalCaddyfile() error = %v, wantErr %v", err, tt.wantErr)
 			}
-			if se.Template != tt.fields.Template || se.Placeholder != tt.fields.Placeholder || !reflect.DeepEqual(se.LogEncoderConfig, tt.fields.LogEncoderConfig) {
-				t.Errorf("Unexpected marshalling error: expected = %+v, received: %+v", tt.fields, *se)
+			if se.Template != tt.fields.Template || se.Placeholder != tt.fields.Placeholder || se.UnescapeStrings != tt.fields.UnescapeStrings || !reflect.DeepEqual(se.LogEncoderConfig, tt.fields.LogEncoderConfig) {
+				t.Fatalf("Unexpected marshalling error: expected = %+v, received: %+v", tt.fields, *se)
 			}
 		})
 	}

formatencoder.go

@@ -64,6 +64,7 @@ type TransformEncoder struct {
 	zapcore.Encoder `json:"-"`
 	Template        string `json:"template,omitempty"`
 	Placeholder     string `json:"placeholder,omitempty"`
+	UnescapeStrings bool   `json:"unescape_strings,omitempty"`
 }
 
 func (TransformEncoder) CaddyModule() caddy.ModuleInfo {
@@ -106,6 +107,7 @@ func (se TransformEncoder) Clone() zapcore.Encoder {
 		Encoder:          se.Encoder.Clone(),
 		Template:         se.Template,
 		Placeholder:      se.Placeholder,
+		UnescapeStrings:  se.UnescapeStrings,
 	}
 }
 
@@ -119,7 +121,7 @@ func (se TransformEncoder) EncodeEntry(ent zapcore.Entry, fields []zapcore.Field
 	repl.Map(func(key string) (interface{}, bool) {
 		if strings.Contains(key, ":") {
 			for _, slice := range strings.Split(key, ":") {
-				val, found := getValue(buf, slice)
+				val, found := getValue(buf, slice, se.UnescapeStrings)
 				if found {
 					return val, found
 				}
@@ -128,7 +130,7 @@ func (se TransformEncoder) EncodeEntry(ent zapcore.Entry, fields []zapcore.Field
 			return nil, false
 		}
 
-		return getValue(buf, key)
+		return getValue(buf, key, se.UnescapeStrings)
 	})
 
 	out := repl.ReplaceAll(se.Template, se.Placeholder)
@@ -144,7 +146,7 @@ func (se TransformEncoder) EncodeEntry(ent zapcore.Entry, fields []zapcore.Field
 	return buf, err
 }
 
-func getValue(buf *buffer.Buffer, key string) (interface{}, bool) {
+func getValue(buf *buffer.Buffer, key string, unescapeStrings bool) (interface{}, bool) {
 	path := strings.Split(key, ">")
 	value, dataType, _, err := jsonparser.Get(buf.Bytes(), path...)
 	if err != nil {
@@ -153,7 +155,14 @@ func getValue(buf *buffer.Buffer, key string) (interface{}, bool) {
 	switch dataType {
 	case jsonparser.NotExist:
 		return nil, false
-	case jsonparser.Array, jsonparser.Boolean, jsonparser.Null, jsonparser.Number, jsonparser.Object, jsonparser.String, jsonparser.Unknown:
+	case jsonparser.String:
+		if !unescapeStrings {
+			return value, true
+		} else {
+			str, _ := jsonparser.ParseString(value)
+			return str, true
+		}
+	case jsonparser.Array, jsonparser.Boolean, jsonparser.Null, jsonparser.Number, jsonparser.Object, jsonparser.Unknown:
 		// if a value exists, return it as is. A byte is a byte is a byte. The replacer handles them just fine.
 		return value, true
 	default:

formatencoder_test.go

@@ -0,0 +1,82 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package transformencoder
+
+import (
+	"testing"
+
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/modules/logging"
+	"github.com/stretchr/testify/assert"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+)
+
+func TestEncodeEntry(t *testing.T) {
+	tests := []struct {
+		name              string
+		se                TransformEncoder
+		entry             zapcore.Entry
+		fields            []zapcore.Field
+		expectedLogString string
+	}{
+		{
+			name: "transform: no args",
+			se: TransformEncoder{
+				Encoder:  new(logging.JSONEncoder),
+				Template: "{msg} {username}",
+			},
+			entry: zapcore.Entry{
+				Message: "lob\nlaw",
+			},
+			fields: []zapcore.Field{
+				zap.String("username", "john\ndoe"),
+			},
+			expectedLogString: "lob\\nlaw john\\ndoe\n",
+		},
+		{
+			name: "transform: no args",
+			se: TransformEncoder{
+				Encoder:         new(logging.JSONEncoder),
+				Template:        "{msg} {username}",
+				UnescapeStrings: true,
+			},
+			entry: zapcore.Entry{
+				Message: "lob\nlaw",
+			},
+			fields: []zapcore.Field{
+				zap.String("username", "john\ndoe"),
+			},
+			expectedLogString: "lob\nlaw john\ndoe\n",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+
+			err := tt.se.Provision(caddy.Context{})
+			assert.NoError(t, err)
+
+			buf, err := tt.se.EncodeEntry(tt.entry, tt.fields)
+
+			if err != nil {
+				t.Errorf("TransformEncoder.EncodeEntry() error = %v", err)
+			}
+
+			assert.Equal(t, tt.expectedLogString, buf.String())
+
+		})
+	}
+}

go.mod

@@ -5,6 +5,7 @@ go 1.18
 require (
 	github.com/buger/jsonparser v1.1.1
 	github.com/caddyserver/caddy/v2 v2.6.2
+	github.com/stretchr/testify v1.10.0
 	go.uber.org/zap v1.23.0
 )
 
@@ -22,6 +23,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgraph-io/badger v1.6.2 // indirect
 	github.com/dgraph-io/badger/v2 v2.2007.4 // indirect
 	github.com/dgraph-io/ristretto v0.0.4-0.20200906165740-41ebdbffecfd // indirect
@@ -69,6 +71,7 @@ require (
 	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/onsi/ginkgo v1.16.4 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.12.2 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.32.1 // indirect
@@ -111,5 +114,6 @@ require (
 	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 	howett.net/plist v1.0.0 // indirect
 )

go.sum

@@ -621,7 +621,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/tailscale/tscert v0.0.0-20220316030059-54bbcb9f74e2 h1:xwMw7LFhV9dbvot9A7NLClP9udqbjrQlIwWMH8e7uiQ=
 github.com/tailscale/tscert v0.0.0-20220316030059-54bbcb9f74e2/go.mod h1:hL4gB6APAasMR2NNi/JHzqKkxW3EPQlFgLEq9PMi2t0=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@@ -1070,6 +1071,7 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=