Skip to content

Commit 3cb3235

Browse files
acabarbayeacabarbaye
authored
✨ Generic map and slice serialisers (#622)
<!-- Copyright (C) 2020-2022 Arm Limited or its affiliates and Contributors. All rights reserved. SPDX-License-Identifier: Apache-2.0 --> ### Description - extended mapstructure decode - add ways to flatten and expand maps ### Test Coverage <!-- Please put an `x` in the correct box e.g. `[x]` to indicate the testing coverage of this change. --> - [x] This change is covered by existing or additional automated tests. - [ ] Manual testing has been performed (and evidence provided) as automated testing was not feasible. - [ ] Additional tests are not required for this change (e.g. documentation update).
1 parent 9c7e5fd commit 3cb3235

34 files changed

+1459
-29
lines changed

‎.secrets.baseline

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -265,5 +265,5 @@
265265
}
266266
]
267267
},
268-
"generated_at": "2025-05-12T17:01:47Z"
268+
"generated_at": "2025-05-30T10:38:21Z"
269269
}

‎changes/20250530113842.bugfix

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
:camel: Upgrade dependencies

‎changes/20250530114030.bugfix

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
:gear: Move dependency on mapstructure to use https://github.com/go-viper/mapstructure as https://github.com/mitchellh/mapstructure is [no longer maintained](https://github.com/go-viper/mapstructure?tab=readme-ov-file#migrating-from-githubcommitchellhmapstructure)

‎changes/20250530114133.feature

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
:sparkles: [serialization] Added ways to serialise structs into maps or slice

‎changes/20250530114405.feature

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
:sparkles: [maps] Added ways to expand and flatten maps in a similar fashion to https://github.com/krakend/flatmap and no longer available https://pkg.go.dev/github.com/hashicorp/terraform/flatmap

‎utils/charset/iconv/interfaces.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import (
1111
"io"
1212
)
1313

14-
//go:generate mockgen -destination=../../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/charset/$GOPACKAGE ICharsetConverter
14+
//go:generate go tool mockgen -destination=../../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/charset/$GOPACKAGE ICharsetConverter
1515

1616
type ICharsetConverter interface {
1717
// ConvertString converts the charset of an input string

‎utils/collection/pagination/interfaces.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import (
1212
"io"
1313
)
1414

15-
//go:generate mockgen -destination=../../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/collection/$GOPACKAGE IStaticPage,IPage,IStaticPageStream,IStream,IIterator,IPaginator,IPaginatorAndPageFetcher,IStreamPaginator,IStreamPaginatorAndPageFetcher
15+
//go:generate go tool mockgen -destination=../../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/collection/$GOPACKAGE IStaticPage,IPage,IStaticPageStream,IStream,IIterator,IPaginator,IPaginatorAndPageFetcher,IStreamPaginator,IStreamPaginatorAndPageFetcher
1616

1717
// IIterator defines an iterator over a collection of items.
1818
type IIterator interface {

‎utils/collection/parseLists.go

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package collection
66

77
import (
88
"fmt"
9+
"slices"
910
"strings"
1011
"unicode"
1112

@@ -75,11 +76,9 @@ func ConvertSliceToMap[T comparable](input []T) (pairs map[T]T, err error) {
7576
}
7677

7778
pairs = make(map[T]T, numElements/2)
78-
// TODO use slices.Chunk introduced in go 23 when library is upgraded
79-
for i := 0; i < numElements; i += 2 {
80-
pairs[input[i]] = input[i+1]
79+
for pair := range slices.Chunk(input, 2) {
80+
pairs[pair[0]] = pair[1]
8181
}
82-
8382
return
8483
}
8584

‎utils/config/interfaces.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
// Package config provides utilities to load configuration from an environment and perform validation at load time.
77
package config
88

9-
//go:generate mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IServiceConfiguration,Validator
9+
//go:generate go tool mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IServiceConfiguration,Validator
1010

1111
// IServiceConfiguration defines a typical service configuration.
1212
type IServiceConfiguration interface {

‎utils/config/service_configuration.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ import (
99
"fmt"
1010
"strings"
1111

12+
"github.com/go-viper/mapstructure/v2"
1213
"github.com/joho/godotenv"
13-
"github.com/mitchellh/mapstructure"
1414
"github.com/spf13/pflag"
1515
"github.com/spf13/viper"
1616

‎utils/encryption/interface.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import (
66
"fmt"
77
)
88

9-
//go:generate mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IKeyPair
9+
//go:generate go tool mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IKeyPair
1010

1111
// IKeyPair defines an asymmetric key pair for cryptography.
1212
type IKeyPair interface {

‎utils/environment/interfaces.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import (
99
"github.com/ARM-software/golang-utils/utils/filesystem"
1010
)
1111

12-
//go:generate mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IEnvironmentVariable,IEnvironment
12+
//go:generate go tool mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IEnvironmentVariable,IEnvironment
1313

1414
// IEnvironmentVariable defines an environment variable to be set for the commands to run.
1515
type IEnvironmentVariable interface {

‎utils/filesystem/interfaces.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ import (
1515
"github.com/ARM-software/golang-utils/utils/config"
1616
)
1717

18-
//go:generate mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IFileHash,IChowner,ILinker,File,DiskUsage,FileTimeInfo,ILock,ILimits,FS,ICloseableFS,IForceRemover,IStater,ILinkReader,ISymLinker
18+
//go:generate go tool mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IFileHash,IChowner,ILinker,File,DiskUsage,FileTimeInfo,ILock,ILimits,FS,ICloseableFS,IForceRemover,IStater,ILinkReader,ISymLinker
1919

2020
// IFileHash defines a file hash.
2121
// For reference.

‎utils/go.mod

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ require (
2020
github.com/go-logr/stdr v1.2.2
2121
github.com/go-logr/zapr v1.3.0
2222
github.com/go-ozzo/ozzo-validation/v4 v4.3.0
23+
github.com/go-viper/mapstructure/v2 v2.2.1
2324
github.com/gofrs/uuid/v5 v5.3.2
2425
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f
2526
github.com/hashicorp/go-cleanhttp v0.5.2
@@ -28,7 +29,6 @@ require (
2829
github.com/iamacarpet/go-win64api v0.0.0-20230324134531-ef6dbdd6db97
2930
github.com/joho/godotenv v1.5.1
3031
github.com/mitchellh/go-homedir v1.1.0
31-
github.com/mitchellh/mapstructure v1.5.0
3232
github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5
3333
github.com/rs/zerolog v1.34.0
3434
github.com/sasha-s/go-deadlock v0.3.5
@@ -45,7 +45,7 @@ require (
4545
go.uber.org/mock v0.5.2
4646
go.uber.org/zap v1.27.0
4747
golang.org/x/crypto v0.38.0
48-
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
48+
golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6
4949
golang.org/x/mod v0.24.0
5050
golang.org/x/net v0.40.0
5151
golang.org/x/oauth2 v0.30.0
@@ -70,7 +70,6 @@ require (
7070
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
7171
github.com/go-git/go-billy/v5 v5.6.2 // indirect
7272
github.com/go-ole/go-ole v1.2.6 // indirect
73-
github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
7473
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
7574
github.com/google/cabbie v1.0.2 // indirect
7675
github.com/google/glazier v0.0.0-20211029225403-9f766cca891d // indirect
@@ -97,7 +96,7 @@ require (
9796
github.com/xanzy/ssh-agent v0.3.3 // indirect
9897
github.com/yusufpapurcu/wmi v1.2.4 // indirect
9998
go.uber.org/multierr v1.10.0 // indirect
100-
golang.org/x/tools v0.30.0 // indirect
99+
golang.org/x/tools v0.33.0 // indirect
101100
gopkg.in/warnings.v0 v0.1.2 // indirect
102101
gopkg.in/yaml.v3 v3.0.1 // indirect
103102
)

‎utils/go.sum

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -159,8 +159,6 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
159159
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
160160
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
161161
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
162-
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
163-
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
164162
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
165163
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
166164
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
@@ -254,8 +252,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
254252
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
255253
golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
256254
golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
257-
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
258-
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
255+
golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI=
256+
golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ=
259257
golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
260258
golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
261259
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -313,8 +311,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
313311
golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
314312
golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
315313
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
316-
golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY=
317-
golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY=
314+
golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc=
315+
golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI=
318316
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
319317
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
320318
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=

‎utils/hashing/interfaces.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import (
1111
"io"
1212
)
1313

14-
//go:generate mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IHash
14+
//go:generate go tool mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IHash
1515

1616
// IHash defines a hashing algorithm.
1717
type IHash interface {

‎utils/http/interfaces.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ import (
2626
"github.com/hashicorp/go-retryablehttp"
2727
)
2828

29-
//go:generate mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IClient,IRetryWaitPolicy,IClientWithHeaders
29+
//go:generate go tool mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE IClient,IRetryWaitPolicy,IClientWithHeaders
3030

3131
// IClient defines an HTTP client similar to http.Client but without shared state with other clients used in the same program.
3232
// See https://github.com/hashicorp/go-cleanhttp for more details.

‎utils/logs/interfaces.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import (
1212
"github.com/go-logr/logr"
1313
)
1414

15-
//go:generate mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE Loggers,IMultipleLoggers,WriterWithSource,StdLogger
15+
//go:generate go tool mockgen -destination=../mocks/mock_$GOPACKAGE.go -package=mocks github.com/ARM-software/golang-utils/utils/$GOPACKAGE Loggers,IMultipleLoggers,WriterWithSource,StdLogger
1616

1717
// Loggers defines generic loggers which separate common logging messages from errors.
1818
// This is to use in cases where it is necessary to separate the two streams e.g. remote procedure call (RPC)

‎utils/maps/ReadME.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
This module was initially created to vendor https://pkg.go.dev/github.com/hashicorp/terraform/flatmap which has been
2+
removed from the terraform project.
3+
4+
code has been updated and is also inspired from https://github.com/astaxie/flatmap

‎utils/maps/expand.go

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
package maps
2+
3+
import (
4+
"fmt"
5+
"sort"
6+
"strconv"
7+
"strings"
8+
)
9+
10+
func Expand(value map[string]string) (expandedMap any, err error) {
11+
if len(value) == 0 {
12+
return
13+
}
14+
m := make(map[string]any, len(value))
15+
for k := range value {
16+
key, _, found := strings.Cut(k, separator)
17+
if found {
18+
subMap, subErr := ExpandPrefixed(value, key)
19+
if subErr != nil {
20+
err = subErr
21+
return
22+
}
23+
m[key] = subMap
24+
} else {
25+
m[k] = value[k]
26+
}
27+
}
28+
expandedMap = m
29+
return
30+
}
31+
32+
// ExpandPrefixed takes a maps and a prefix and expands that value into
33+
// a more complex structure. This is the reverse of the Flatten operation.
34+
func ExpandPrefixed(m map[string]string, key string) (result any, err error) {
35+
// If the key is exactly a key in the maps, just return it
36+
if v, ok := m[key]; ok {
37+
if v == "true" {
38+
result = true
39+
return
40+
} else if v == "false" {
41+
result = false
42+
return
43+
}
44+
45+
result = v
46+
return
47+
}
48+
49+
// Check if the key is an array, and if so, expand the array
50+
arrayKey := fmt.Sprintf("%v%v%d", key, separator, 0)
51+
if _, ok := m[arrayKey]; ok {
52+
result, err = expandArray(m, key)
53+
return
54+
}
55+
arrayKey = fmt.Sprintf("%v%v", arrayKey, separator)
56+
for k := range m {
57+
if strings.HasPrefix(k, arrayKey) {
58+
result, err = expandArray(m, key)
59+
return
60+
}
61+
}
62+
63+
// Check if this is a prefix in the maps
64+
prefix := key + separator
65+
for k := range m {
66+
if strings.HasPrefix(k, prefix) {
67+
result, err = expandMap(m, prefix)
68+
return
69+
}
70+
}
71+
72+
result = nil
73+
return
74+
}
75+
76+
func expandArray(m map[string]string, prefix string) (result []any, err error) {
77+
keySet := map[int]bool{}
78+
for k := range m {
79+
if !strings.HasPrefix(k, prefix+separator) {
80+
continue
81+
}
82+
83+
key := k[len(prefix)+1:]
84+
idx := strings.Index(key, separator)
85+
if idx != -1 {
86+
key = key[:idx]
87+
}
88+
89+
k, subErr := strconv.Atoi(key)
90+
if subErr != nil {
91+
err = subErr
92+
return
93+
}
94+
keySet[k] = true
95+
}
96+
97+
var keysList []int
98+
for key := range keySet {
99+
keysList = append(keysList, key)
100+
}
101+
sort.Ints(keysList)
102+
103+
r := make([]any, len(keysList))
104+
for i, key := range keysList {
105+
keyString := strconv.Itoa(key)
106+
item, subErr := ExpandPrefixed(m, fmt.Sprintf("%s.%s", prefix, keyString))
107+
r[i] = item
108+
if subErr != nil {
109+
err = subErr
110+
return
111+
}
112+
}
113+
result = r
114+
return
115+
}
116+
117+
func expandMap(m map[string]string, prefix string) (r map[string]any, err error) {
118+
result := make(map[string]any)
119+
for k := range m {
120+
if !strings.HasPrefix(k, prefix) {
121+
continue
122+
}
123+
124+
key := k[len(prefix):]
125+
idx := strings.Index(key, separator)
126+
if idx != -1 {
127+
key = key[:idx]
128+
}
129+
if _, ok := result[key]; ok {
130+
continue
131+
}
132+
133+
item, subErr := ExpandPrefixed(m, k[:len(prefix)+len(key)])
134+
if subErr != nil {
135+
err = subErr
136+
return
137+
}
138+
result[key] = item
139+
}
140+
141+
r = result
142+
return
143+
}

0 commit comments

Comments
 (0)