Skip to content

✨ Generic map and slice serialisers #622

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
May 30, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .secrets.baseline
Original file line number Diff line number Diff line change
Expand Up @@ -265,5 +265,5 @@
}
]
},
"generated_at": "2025-05-12T17:01:47Z"
"generated_at": "2025-05-30T10:38:21Z"
}
1 change: 1 addition & 0 deletions changes/20250530113842.bugfix
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
:camel: Upgrade dependencies
1 change: 1 addition & 0 deletions changes/20250530114030.bugfix
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
:gear: Move dependency on mapstructure to use https://github.com/go-viper/mapstructure as https://github.com/mitchellh/mapstructure is [no longer maintained](https://github.com/go-viper/mapstructure?tab=readme-ov-file#migrating-from-githubcommitchellhmapstructure)
1 change: 1 addition & 0 deletions changes/20250530114133.feature
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
:sparkles: [serialization] Added ways to serialise structs into maps or slice
1 change: 1 addition & 0 deletions changes/20250530114405.feature
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
:sparkles: [maps] Added ways to expand and flatten maps in a similar fashion to https://github.com/krakend/flatmap and no longer available https://pkg.go.dev/github.com/hashicorp/terraform/flatmap
2 changes: 1 addition & 1 deletion utils/charset/iconv/interfaces.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (
"io"
)

//go:generate mockgen -destination=../../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/charset/$GOPACKAGE ICharsetConverter
//go:generate go tool mockgen -destination=../../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/charset/$GOPACKAGE ICharsetConverter

type ICharsetConverter interface {
// ConvertString converts the charset of an input string
Expand Down
2 changes: 1 addition & 1 deletion utils/collection/pagination/interfaces.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (
"io"
)

//go:generate mockgen -destination=../../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/collection/$GOPACKAGE IStaticPage,IPage,IStaticPageStream,IStream,IIterator,IPaginator,IPaginatorAndPageFetcher,IStreamPaginator,IStreamPaginatorAndPageFetcher
//go:generate go tool mockgen -destination=../../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/collection/$GOPACKAGE IStaticPage,IPage,IStaticPageStream,IStream,IIterator,IPaginator,IPaginatorAndPageFetcher,IStreamPaginator,IStreamPaginatorAndPageFetcher

// IIterator defines an iterator over a collection of items.
type IIterator interface {
Expand Down
7 changes: 3 additions & 4 deletions utils/collection/parseLists.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package collection

import (
"fmt"
"slices"
"strings"
"unicode"

Expand Down Expand Up @@ -75,11 +76,9 @@ func ConvertSliceToMap[T comparable](input []T) (pairs map[T]T, err error) {
}

pairs = make(map[T]T, numElements/2)
// TODO use slices.Chunk introduced in go 23 when library is upgraded
for i := 0; i < numElements; i += 2 {
pairs[input[i]] = input[i+1]
for pair := range slices.Chunk(input, 2) {
pairs[pair[0]] = pair[1]
}

return
}

Expand Down
2 changes: 1 addition & 1 deletion utils/config/interfaces.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
// Package config provides utilities to load configuration from an environment and perform validation at load time.
package config

//go:generate mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IServiceConfiguration,Validator
//go:generate go tool mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IServiceConfiguration,Validator

// IServiceConfiguration defines a typical service configuration.
type IServiceConfiguration interface {
Expand Down
2 changes: 1 addition & 1 deletion utils/config/service_configuration.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ import (
"fmt"
"strings"

"github.com/go-viper/mapstructure/v2"
"github.com/joho/godotenv"
"github.com/mitchellh/mapstructure"
"github.com/spf13/pflag"
"github.com/spf13/viper"

Expand Down
2 changes: 1 addition & 1 deletion utils/encryption/interface.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import (
"fmt"
)

//go:generate mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IKeyPair
//go:generate go tool mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IKeyPair

// IKeyPair defines an asymmetric key pair for cryptography.
type IKeyPair interface {
Expand Down
2 changes: 1 addition & 1 deletion utils/environment/interfaces.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
"github.com/ARM-software/golang-utils/utils/filesystem"
)

//go:generate mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IEnvironmentVariable,IEnvironment
//go:generate go tool mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IEnvironmentVariable,IEnvironment

// IEnvironmentVariable defines an environment variable to be set for the commands to run.
type IEnvironmentVariable interface {
Expand Down
2 changes: 1 addition & 1 deletion utils/filesystem/interfaces.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import (
"github.com/ARM-software/golang-utils/utils/config"
)

//go:generate mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IFileHash,IChowner,ILinker,File,DiskUsage,FileTimeInfo,ILock,ILimits,FS,ICloseableFS,IForceRemover,IStater,ILinkReader,ISymLinker
//go:generate go tool mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IFileHash,IChowner,ILinker,File,DiskUsage,FileTimeInfo,ILock,ILimits,FS,ICloseableFS,IForceRemover,IStater,ILinkReader,ISymLinker

// IFileHash defines a file hash.
// For reference.
Expand Down
7 changes: 3 additions & 4 deletions utils/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ require (
github.com/go-logr/stdr v1.2.2
github.com/go-logr/zapr v1.3.0
github.com/go-ozzo/ozzo-validation/v4 v4.3.0
github.com/go-viper/mapstructure/v2 v2.2.1
github.com/gofrs/uuid/v5 v5.3.2
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f
github.com/hashicorp/go-cleanhttp v0.5.2
Expand All @@ -28,7 +29,6 @@ require (
github.com/iamacarpet/go-win64api v0.0.0-20230324134531-ef6dbdd6db97
github.com/joho/godotenv v1.5.1
github.com/mitchellh/go-homedir v1.1.0
github.com/mitchellh/mapstructure v1.5.0
github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5
github.com/rs/zerolog v1.34.0
github.com/sasha-s/go-deadlock v0.3.5
Expand All @@ -45,7 +45,7 @@ require (
go.uber.org/mock v0.5.2
go.uber.org/zap v1.27.0
golang.org/x/crypto v0.38.0
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6
golang.org/x/mod v0.24.0
golang.org/x/net v0.40.0
golang.org/x/oauth2 v0.30.0
Expand All @@ -70,7 +70,6 @@ require (
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-git/go-billy/v5 v5.6.2 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
github.com/google/cabbie v1.0.2 // indirect
github.com/google/glazier v0.0.0-20211029225403-9f766cca891d // indirect
Expand All @@ -97,7 +96,7 @@ require (
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.uber.org/multierr v1.10.0 // indirect
golang.org/x/tools v0.30.0 // indirect
golang.org/x/tools v0.33.0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
Expand Down
10 changes: 4 additions & 6 deletions utils/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -159,8 +159,6 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
Expand Down Expand Up @@ -254,8 +252,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI=
golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ=
golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
Expand Down Expand Up @@ -313,8 +311,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY=
golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY=
golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc=
golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
Expand Down
2 changes: 1 addition & 1 deletion utils/hashing/interfaces.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (
"io"
)

//go:generate mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IHash
//go:generate go tool mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IHash

// IHash defines a hashing algorithm.
type IHash interface {
Expand Down
2 changes: 1 addition & 1 deletion utils/http/interfaces.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import (
"github.com/hashicorp/go-retryablehttp"
)

//go:generate mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IClient,IRetryWaitPolicy,IClientWithHeaders
//go:generate go tool mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IClient,IRetryWaitPolicy,IClientWithHeaders

// IClient defines an HTTP client similar to http.Client but without shared state with other clients used in the same program.
// See https://github.com/hashicorp/go-cleanhttp for more details.
Expand Down
2 changes: 1 addition & 1 deletion utils/logs/interfaces.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (
"github.com/go-logr/logr"
)

//go:generate mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE Loggers,IMultipleLoggers,WriterWithSource,StdLogger
//go:generate go tool mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE Loggers,IMultipleLoggers,WriterWithSource,StdLogger

// Loggers defines generic loggers which separate common logging messages from errors.
// This is to use in cases where it is necessary to separate the two streams e.g. remote procedure call (RPC)
Expand Down
4 changes: 4 additions & 0 deletions utils/maps/ReadME.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
This module was initially created to vendor https://pkg.go.dev/github.com/hashicorp/terraform/flatmap which has been
removed from the terraform project.

code has been updated and is also inspired from https://github.com/astaxie/flatmap
143 changes: 143 additions & 0 deletions utils/maps/expand.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
package maps

import (
"fmt"
"sort"
"strconv"
"strings"
)

func Expand(value map[string]string) (expandedMap any, err error) {
if len(value) == 0 {
return
}
m := make(map[string]any, len(value))
for k := range value {
key, _, found := strings.Cut(k, separator)
if found {
subMap, subErr := ExpandPrefixed(value, key)
if subErr != nil {
err = subErr
return
}
m[key] = subMap
} else {
m[k] = value[k]
}
}
expandedMap = m
return
}

// ExpandPrefixed takes a maps and a prefix and expands that value into
// a more complex structure. This is the reverse of the Flatten operation.
func ExpandPrefixed(m map[string]string, key string) (result any, err error) {
// If the key is exactly a key in the maps, just return it
if v, ok := m[key]; ok {
if v == "true" {
result = true
return
} else if v == "false" {
result = false
return
}

result = v
return
}

// Check if the key is an array, and if so, expand the array
arrayKey := fmt.Sprintf("%v%v%d", key, separator, 0)
if _, ok := m[arrayKey]; ok {
result, err = expandArray(m, key)
return
}
arrayKey = fmt.Sprintf("%v%v", arrayKey, separator)
for k := range m {
if strings.HasPrefix(k, arrayKey) {
result, err = expandArray(m, key)
return
}
}

// Check if this is a prefix in the maps
prefix := key + separator
for k := range m {
if strings.HasPrefix(k, prefix) {
result, err = expandMap(m, prefix)
return
}
}

result = nil
return
}

func expandArray(m map[string]string, prefix string) (result []any, err error) {
keySet := map[int]bool{}
for k := range m {
if !strings.HasPrefix(k, prefix+separator) {
continue
}

key := k[len(prefix)+1:]
idx := strings.Index(key, separator)
if idx != -1 {
key = key[:idx]
}

k, subErr := strconv.Atoi(key)
if subErr != nil {
err = subErr
return
}
keySet[k] = true
}

var keysList []int
for key := range keySet {
keysList = append(keysList, key)
}
sort.Ints(keysList)

r := make([]any, len(keysList))
for i, key := range keysList {
keyString := strconv.Itoa(key)
item, subErr := ExpandPrefixed(m, fmt.Sprintf("%s.%s", prefix, keyString))
r[i] = item
if subErr != nil {
err = subErr
return
}
}
result = r
return
}

func expandMap(m map[string]string, prefix string) (r map[string]any, err error) {
result := make(map[string]any)
for k := range m {
if !strings.HasPrefix(k, prefix) {
continue
}

key := k[len(prefix):]
idx := strings.Index(key, separator)
if idx != -1 {
key = key[:idx]
}
if _, ok := result[key]; ok {
continue
}

item, subErr := ExpandPrefixed(m, k[:len(prefix)+len(key)])
if subErr != nil {
err = subErr
return
}
result[key] = item
}

r = result
return
}
Loading
Loading