kingpin to kong

Author: everesio

CHANGELOG.md

@@ -1,5 +1,9 @@
 ## Changelog
 
+### [v0.1.1](https://github.com/grepplabs/mqtt-proxy/releases/tag/v0.1.1) - 6 November 2022
+* Update dependencies
+* Migrate from [kingpin](https://github.com/alecthomas/kingpin) to [kong](https://github.com/alecthomas/kong)
+
 ### [v0.1.0](https://github.com/grepplabs/mqtt-proxy/releases/tag/v0.1.0) - 1 November 2022
 * Add server certificates rotation
 * Add client certificate revocation list

Makefile

@@ -2,7 +2,7 @@
 
 .PHONY: clean build fmt test
 
-TAG           ?= v0.1.0
+TAG           ?= v0.1.1
 
 BUILD_FLAGS   ?=
 BINARY        ?= mqtt-proxy

cmd/root.go

@@ -5,47 +5,52 @@ import (
 	"fmt"
 	"os"
 	"os/signal"
-	"path/filepath"
 	"syscall"
 
+	"github.com/alecthomas/kong"
+	kongyaml "github.com/alecthomas/kong-yaml"
+	"github.com/grepplabs/mqtt-proxy/pkg/config"
 	"github.com/grepplabs/mqtt-proxy/pkg/log"
 	"github.com/oklog/run"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/collectors"
 	"github.com/prometheus/common/version"
 	"go.uber.org/automaxprocs/maxprocs"
-	"gopkg.in/alecthomas/kingpin.v2"
 )
 
 type setupFunc func(*run.Group, log.Logger, *prometheus.Registry) error
 
-func Execute() {
-	app := kingpin.New(filepath.Base(os.Args[0]), "MQTT Proxy")
-
-	app.Version(version.Print("mqtt-proxy"))
-	app.HelpFlag.Short('h')
-
-	logConfig := log.LogConfig{}
-	app.Flag("log.level", "Log filtering One of: [fatal, error, warn, info, debug]").Default(log.Info).EnumVar(&logConfig.LogLevel, log.Fatal, log.Error, log.Warn, log.Info, log.Debug)
-	app.Flag("log.format", "Log format to use. One of: [logfmt, json, plain]").Default(log.LogFormatLogfmt).EnumVar(&logConfig.LogFormat, log.LogFormatLogfmt, log.LogFormatJson, log.LogFormatPlain)
-	app.Flag("log.field-name.time", "Log time field name").Default(log.TimeKey).StringVar(&logConfig.LogFieldNames.Time)
-	app.Flag("log.field-name.message", "Log message field name").Default(log.MessageKey).StringVar(&logConfig.LogFieldNames.Message)
-	app.Flag("log.field-name.error", "Log error field name").Default(log.ErrorKey).StringVar(&logConfig.LogFieldNames.Error)
-	app.Flag("log.field-name.caller", "Log caller field name").Default(log.CallerKey).StringVar(&logConfig.LogFieldNames.Caller)
-	app.Flag("log.field-name.level", "Log time field name").Default(log.LevelKey).StringVar(&logConfig.LogFieldNames.Level)
+type CLI struct {
+	LogConfig log.Config    `embed:"" prefix:"log."`
+	Server    config.Server `name:"server" cmd:"" help:"MQTT Proxy"`
+	Version   struct{}      `name:"version" cmd:"" help:"Version information"`
+}
 
+func Execute() {
 	cmds := map[string]setupFunc{}
 
-	registerServer(cmds, app)
-
-	cmd, err := app.Parse(os.Args[1:])
-	if err != nil {
-		_, _ = fmt.Fprintln(os.Stderr, fmt.Errorf("error parsing commandline arguments: %w", err))
-		app.Usage(os.Args[1:])
-		os.Exit(2)
+	var cli CLI
+	ctx := kong.Parse(&cli,
+		kong.Name(os.Args[0]),
+		kong.Description("MQTT Proxy"),
+		kong.Configuration(kong.JSON, "/etc/mqtt-proxy/config.json", "~/.mqtt-proxy.json"),
+		kong.Configuration(kongyaml.Loader, "/etc/mqtt-proxy/config.yaml", "~/.mqtt-proxy.yaml"),
+		kong.UsageOnError(),
+		log.Vars(), config.ServerVars())
+	switch ctx.Command() {
+	case "server":
+		cmds[ctx.Command()] = func(group *run.Group, logger log.Logger, registry *prometheus.Registry) error {
+			return runServer(group, logger, registry, &cli.Server)
+		}
+	case "version":
+		fmt.Println(version.Print("mqtt-proxy"))
+		os.Exit(0)
+	default:
+		fmt.Println(ctx.Command())
+		os.Exit(1)
 	}
 
-	logger := log.NewLogger(logConfig)
+	logger := log.NewLogger(cli.LogConfig)
 	log.InitInstance(logger)
 
 	undo, err := maxprocs.Set(maxprocs.Logger(func(template string, args ...interface{}) {
@@ -68,9 +73,9 @@ func Execute() {
 	)
 	var g run.Group
 
-	if err := cmds[cmd](&g, logger, metrics); err != nil {
+	if err := cmds[ctx.Command()](&g, logger, metrics); err != nil {
 		// Use %+v for github.com/pkg/errors error to print with stack.
-		logger.WithError(err).Fatalf("preparing %s command failed", cmd)
+		logger.WithError(err).Fatalf("preparing %s command failed", ctx.Command())
 	}
 	{
 		cancel := make(chan struct{})
@@ -81,7 +86,7 @@ func Execute() {
 		})
 	}
 	if err := g.Run(); err != nil {
-		logger.WithError(err).Fatalf("%s command failed", cmd)
+		logger.WithError(err).Fatalf("%s command failed", ctx.Command())
 	}
 	logger.Infof("exiting")
 }

cmd/root_test.go

@@ -0,0 +1,87 @@
+package cmd
+
+import (
+	"github.com/alecthomas/kong"
+	"github.com/confluentinc/confluent-kafka-go/kafka"
+	"github.com/grepplabs/mqtt-proxy/pkg/config"
+	"github.com/grepplabs/mqtt-proxy/pkg/log"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestDefaultServerConfig(t *testing.T) {
+	testCLI, command, err := parseTestCLI([]string{"server"})
+	require.NoError(t, err)
+	require.Equal(t, "server", command)
+	require.Nil(t, testCLI.Server.MQTT.Handler.Authenticator.Plain.Credentials)
+	require.Nil(t, testCLI.Server.MQTT.Publisher.Kafka.ConfArgs.ConfigMap())
+	require.Nil(t, testCLI.Server.MQTT.Publisher.Kafka.TopicMappings.Mappings)
+	require.Equal(t, 1, testCLI.Server.MQTT.Publisher.Kafka.Workers)
+}
+
+func TestKafkaConfigOneParam(t *testing.T) {
+	testCLI, _, err := parseTestCLI([]string{
+		"server",
+		"--mqtt.publisher.kafka.config", "producer.sasl.mechanisms=PLAIN,producer.security.protocol=SASL_SSL,producer.sasl.username=myuser,producer.sasl.password=mypasswd",
+	})
+	require.NoError(t, err)
+	require.EqualValues(t, map[string]kafka.ConfigValue{
+		"producer.sasl.mechanisms":   "PLAIN",
+		"producer.security.protocol": "SASL_SSL",
+		"producer.sasl.username":     "myuser",
+		"producer.sasl.password":     "mypasswd",
+	}, testCLI.Server.MQTT.Publisher.Kafka.ConfArgs.ConfigMap())
+}
+
+func TestKafkaConfigMultipleParams(t *testing.T) {
+	testCLI, _, err := parseTestCLI([]string{
+		"server",
+		"--mqtt.publisher.kafka.config", "producer.sasl.mechanisms=PLAIN,producer.security.protocol=SASL_SSL",
+		"--mqtt.publisher.kafka.config", "producer.sasl.username=myuser,producer.sasl.password=mypasswd",
+	})
+	require.NoError(t, err)
+	require.EqualValues(t, map[string]kafka.ConfigValue{
+		"producer.sasl.mechanisms":   "PLAIN",
+		"producer.security.protocol": "SASL_SSL",
+		"producer.sasl.username":     "myuser",
+		"producer.sasl.password":     "mypasswd",
+	}, testCLI.Server.MQTT.Publisher.Kafka.ConfArgs.ConfigMap())
+}
+
+func TestTopicMappingConfig(t *testing.T) {
+	testCLI, _, err := parseTestCLI([]string{
+		"server",
+		"--mqtt.publisher.kafka.topic-mappings", "temperature=temperature, humidity=.*/humidity,brightness=.*brightness, temperature=^cool$",
+	})
+	require.NoError(t, err)
+	require.Equal(t, 4, len(testCLI.Server.MQTT.Publisher.Kafka.TopicMappings.Mappings))
+}
+
+func TestPlainCredentialsConfig(t *testing.T) {
+	testCLI, _, err := parseTestCLI([]string{
+		"server",
+		"--mqtt.handler.auth.plain.credentials", "alice=test1",
+		"--mqtt.handler.auth.plain.credentials", "bob=test2",
+	})
+	require.NoError(t, err)
+	require.EqualValues(t, map[string]string{
+		"alice": "test1",
+		"bob":   "test2",
+	}, testCLI.Server.MQTT.Handler.Authenticator.Plain.Credentials)
+}
+
+func parseTestCLI(args []string) (*CLI, string, error) {
+	testCLI := &CLI{}
+	parser, err := kong.New(testCLI,
+		kong.Name("mqtt-proxy"),
+		kong.Description("MQTT Proxy"),
+		log.Vars(), config.ServerVars())
+	if err != nil {
+		return nil, "", err
+	}
+	command, err := parser.Parse(args)
+	if err != nil {
+		return nil, "", err
+	}
+	return testCLI, command.Command(), nil
+}

cmd/server.go

@@ -23,62 +23,8 @@ import (
 	"github.com/oklog/run"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
-	"gopkg.in/alecthomas/kingpin.v2"
 )
 
-func registerServer(m map[string]setupFunc, app *kingpin.Application) {
-	command := "server"
-
-	cmd := app.Command(command, "mqtt-proxy server")
-
-	cfg := new(config.Server)
-	cfg.Init()
-
-	cmd.Flag("http.listen-address", "Listen host:port for HTTP endpoints.").Default("0.0.0.0:9090").StringVar(&cfg.HTTP.ListenAddress)
-	cmd.Flag("http.grace-period", "Time to wait after an interrupt received for HTTP Server.").Default("10s").DurationVar(&cfg.HTTP.GracePeriod)
-
-	cmd.Flag("mqtt.listen-address", "Listen host:port for MQTT endpoints.").Default("0.0.0.0:1883").StringVar(&cfg.MQTT.ListenAddress)
-	cmd.Flag("mqtt.grace-period", "Time to wait after an interrupt received for MQTT Server.").Default("10s").DurationVar(&cfg.MQTT.GracePeriod)
-	cmd.Flag("mqtt.read-timeout", "Maximum duration for reading the entire request.").Default("5s").DurationVar(&cfg.MQTT.ReadTimeout)
-	cmd.Flag("mqtt.write-timeout", "Maximum duration before timing out writes of the response.").Default("5s").DurationVar(&cfg.MQTT.WriteTimeout)
-	cmd.Flag("mqtt.idle-timeout", "Maximum duration before timing out writes of the response.").Default("0s").DurationVar(&cfg.MQTT.IdleTimeout)
-
-	cmd.Flag("mqtt.reader-buffer-size", "Read buffer size pro tcp connection.").Default("1024").IntVar(&cfg.MQTT.ReaderBufferSize)
-	cmd.Flag("mqtt.writer-buffer-size", "Write buffer size pro tcp connection.").Default("1024").IntVar(&cfg.MQTT.WriterBufferSize)
-
-	cmd.Flag("mqtt.server-tls.enable", "Enable server side TLS").Default("false").BoolVar(&cfg.MQTT.TLSSrv.Enable)
-	cmd.Flag("mqtt.server-tls.cert-source", "TLS certificate source").Default(config.CertSourceFile).EnumVar(&cfg.MQTT.TLSSrv.CertSource, config.CertSourceFile)
-	cmd.Flag("mqtt.server-tls.refresh", "Option to specify the refresh interval for the TLS certificates.").Default("0s").DurationVar(&cfg.MQTT.TLSSrv.Refresh)
-
-	cmd.Flag("mqtt.server-tls.file.cert", "TLS Certificate for MQTT server, leave blank to disable TLS").Default("").StringVar(&cfg.MQTT.TLSSrv.File.Cert)
-	cmd.Flag("mqtt.server-tls.file.key", "TLS Key for the MQTT server, leave blank to disable TLS").Default("").StringVar(&cfg.MQTT.TLSSrv.File.Key)
-	cmd.Flag("mqtt.server-tls.file.client-ca", "TLS CA to verify clients against. If no client CA is specified, there is no client verification on server side.").Default("").StringVar(&cfg.MQTT.TLSSrv.File.ClientCA)
-	cmd.Flag("mqtt.server-tls.file.client-clr", "TLS X509 CLR signed be the client CA. If no revocation list is specified, only client CA is verified").Default("").StringVar(&cfg.MQTT.TLSSrv.File.ClientCLR)
-
-	cmd.Flag("mqtt.handler.ignore-unsupported", "List of unsupported messages which are ignored. One of: [SUBSCRIBE, UNSUBSCRIBE]").PlaceHolder("MSG").EnumsVar(&cfg.MQTT.Handler.IgnoreUnsupported, "SUBSCRIBE", "UNSUBSCRIBE")
-	cmd.Flag("mqtt.handler.allow-unauthenticated", "List of messages for which connection is not disconnected if unauthenticated request is received. One of: [PUBLISH, PUBREL, PINGREQ]").PlaceHolder("MSG").EnumsVar(&cfg.MQTT.Handler.AllowUnauthenticated, "PUBLISH", "PUBREL", "PINGREQ")
-	cmd.Flag("mqtt.handler.publish.timeout", "Maximum duration of sending publish request to broker.").Default("0s").DurationVar(&cfg.MQTT.Handler.Publish.Timeout)
-	cmd.Flag("mqtt.handler.publish.async.at-most-once", "Async publish for AT_MOST_ONCE QoS.").Default("false").BoolVar(&cfg.MQTT.Handler.Publish.Async.AtMostOnce)
-	cmd.Flag("mqtt.handler.publish.async.at-least-once", "Async publish for AT_LEAST_ONCE QoS.").Default("false").BoolVar(&cfg.MQTT.Handler.Publish.Async.AtLeastOnce)
-	cmd.Flag("mqtt.handler.publish.async.exactly-once", "Async publish for EXACTLY_ONCE QoS.").Default("false").BoolVar(&cfg.MQTT.Handler.Publish.Async.ExactlyOnce)
-
-	cmd.Flag("mqtt.handler.auth.name", "Authenticator name. One of: [noop, plain]").Default(config.AuthNoop).EnumVar(&cfg.MQTT.Handler.Authenticator.Name, config.AuthNoop, config.AuthPlain)
-	cmd.Flag("mqtt.handler.auth.plain.credentials", "List of username and password fields.").Default("USERNAME=PASSWORD").StringMapVar(&cfg.MQTT.Handler.Authenticator.Plain.Credentials)
-	cmd.Flag("mqtt.handler.auth.plain.credentials-file", "Location of a headerless CSV file containing `usernanme,password` records").Default("").StringVar(&cfg.MQTT.Handler.Authenticator.Plain.CredentialsFile)
-
-	cmd.Flag("mqtt.publisher.name", "Publisher name. One of: [noop, kafka]").Default(config.PublisherNoop).EnumVar(&cfg.MQTT.Publisher.Name, config.PublisherNoop, config.PublisherKafka)
-	cmd.Flag("mqtt.publisher.kafka.config", "Comma separated list of properties").PlaceHolder("PROP=VAL").SetValue(&cfg.MQTT.Publisher.Kafka.ConfArgs)
-	cmd.Flag("mqtt.publisher.kafka.bootstrap-servers", "Kafka bootstrap servers").Default("localhost:9092").StringVar(&cfg.MQTT.Publisher.Kafka.BootstrapServers)
-	cmd.Flag("mqtt.publisher.kafka.grace-period", "Time to wait after an interrupt received for Kafka publisher.").Default("10s").DurationVar(&cfg.MQTT.Publisher.Kafka.GracePeriod)
-	cmd.Flag("mqtt.publisher.kafka.default-topic", "Default Kafka topic for MQTT publish messages").Default("").StringVar(&cfg.MQTT.Publisher.Kafka.DefaultTopic)
-	cmd.Flag("mqtt.publisher.kafka.topic-mappings", "Comma separated list of Kafka topic to MQTT topic mappings").PlaceHolder("TOPIC=REGEX").SetValue(&cfg.MQTT.Publisher.Kafka.TopicMappings)
-	cmd.Flag("mqtt.publisher.kafka.workers", "Number of kafka publisher workers").Default("1").IntVar(&cfg.MQTT.Publisher.Kafka.Workers)
-
-	m[command] = func(group *run.Group, logger log.Logger, registry *prometheus.Registry) error {
-		return runServer(group, logger, registry, cfg)
-	}
-}
-
 func runServer(
 	group *run.Group,
 	logger log.Logger,

go.mod

@@ -3,7 +3,10 @@ module github.com/grepplabs/mqtt-proxy
 go 1.19
 
 require (
+	github.com/alecthomas/kong v0.7.0
+	github.com/alecthomas/kong-yaml v0.1.1
 	github.com/confluentinc/confluent-kafka-go v1.9.2
+	github.com/go-playground/validator/v10 v10.11.1
 	github.com/oklog/run v1.1.0
 	github.com/prometheus/client_golang v1.13.1
 	github.com/prometheus/common v0.37.0
@@ -12,23 +15,26 @@ require (
 	go.uber.org/atomic v1.10.0
 	go.uber.org/automaxprocs v1.5.1
 	go.uber.org/zap v1.23.0
-	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 )
 
 require (
-	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
-	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/go-playground/locales v0.14.0 // indirect
+	github.com/go-playground/universal-translator v0.18.0 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/leodido/go-urn v1.2.1 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
-	golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664 // indirect
+	golang.org/x/crypto v0.1.0 // indirect
+	golang.org/x/sys v0.1.0 // indirect
+	golang.org/x/text v0.4.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )