diff --git a/go.mod b/go.mod index b4162b5f..d490e940 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( golang.org/x/oauth2 v0.25.0 golang.org/x/sync v0.11.0 google.golang.org/grpc v1.70.0 - gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.31.0 k8s.io/apimachinery v0.31.0 k8s.io/client-go v0.31.0 @@ -154,7 +154,7 @@ require ( gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/apiextensions-apiserver v0.31.2 // indirect k8s.io/apiserver v0.31.0 // indirect k8s.io/cli-runtime v0.31.0 // indirect diff --git a/pkg/argocd/update.go b/pkg/argocd/update.go index 58548b5c..562fa5a6 100644 --- a/pkg/argocd/update.go +++ b/pkg/argocd/update.go @@ -1,6 +1,7 @@ package argocd import ( + "bytes" "context" "fmt" "path/filepath" @@ -21,7 +22,7 @@ import ( "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) // Stores some statistics about the results of a run @@ -418,6 +419,45 @@ func setAppImage(app *v1alpha1.Application, img *image.ContainerImage) error { return err } +func marshalWithIndent(in interface{}, indent int) (out []byte, err error) { + var b bytes.Buffer + encoder := yaml.NewEncoder(&b) + defer encoder.Close() + // note: yaml.v3 will only respect indents from 1 to 9 inclusive. + encoder.SetIndent(indent) + if err = encoder.Encode(in); err != nil { + return nil, err + } + if err = encoder.Close(); err != nil { + return nil, err + } + return b.Bytes(), nil +} + +func guessIndent(root *yaml.Node) int { + node := root + if root.Kind == yaml.DocumentNode { + if len(node.Content) == 0 { + return 2 + } + node = root.Content[0] + } + // anything other than a map at the root makes guessing difficult + if node.Kind != yaml.MappingNode || len(node.Content) == 0 { + return 2 + } + // first level map entries that are themselves mappings or sequences, + // in block style, and are indented, allow guessing the preferred indent. + for i, child := range node.Content { + if i%2 == 1 && child.Column > 1 && child.Column < 10 && child.Style != yaml.FlowStyle { + if child.Kind == yaml.MappingNode || child.Kind == yaml.SequenceNode { + return child.Column - 1 + } + } + } + return 2 +} + // marshalParamsOverride marshals the parameter overrides of a given application // into YAML bytes func marshalParamsOverride(app *v1alpha1.Application, originalData []byte) ([]byte, error) { @@ -441,16 +481,16 @@ func marshalParamsOverride(app *v1alpha1.Application, originalData []byte) ([]by } if len(originalData) == 0 { - override, err = yaml.Marshal(newParams) + override, err = marshalWithIndent(newParams, 2) break } err = yaml.Unmarshal(originalData, ¶ms) if err != nil { - override, err = yaml.Marshal(newParams) + override, err = marshalWithIndent(newParams, 2) break } mergeKustomizeOverride(¶ms, &newParams) - override, err = yaml.Marshal(params) + override, err = marshalWithIndent(params, 2) case ApplicationTypeHelm: if appSource.Helm == nil { return []byte{}, nil @@ -459,11 +499,12 @@ func marshalParamsOverride(app *v1alpha1.Application, originalData []byte) ([]by if strings.HasPrefix(app.Annotations[common.WriteBackTargetAnnotation], common.HelmPrefix) { images := GetImagesAndAliasesFromApplication(app) - helmNewValues := yaml.MapSlice{} + helmNewValues := yaml.Node{} err = yaml.Unmarshal(originalData, &helmNewValues) if err != nil { return nil, err } + indent := guessIndent(&helmNewValues) for _, c := range images { if c.ImageAlias == "" { @@ -505,7 +546,7 @@ func marshalParamsOverride(app *v1alpha1.Application, originalData []byte) ([]by } } - override, err = yaml.Marshal(helmNewValues) + override, err = marshalWithIndent(&helmNewValues, indent) } else { var params helmOverride newParams := helmOverride{ @@ -518,16 +559,16 @@ func marshalParamsOverride(app *v1alpha1.Application, originalData []byte) ([]by log.WithContext().AddField("application", app).Debugf("values: '%s'", outputParams) if len(originalData) == 0 { - override, err = yaml.Marshal(newParams) + override, err = marshalWithIndent(newParams, 2) break } err = yaml.Unmarshal(originalData, ¶ms) if err != nil { - override, err = yaml.Marshal(newParams) + override, err = marshalWithIndent(newParams, 2) break } mergeHelmOverride(¶ms, &newParams) - override, err = yaml.Marshal(params) + override, err = marshalWithIndent(params, 2) } default: err = fmt.Errorf("unsupported application type") @@ -572,72 +613,98 @@ func mergeKustomizeOverride(t *kustomizeOverride, o *kustomizeOverride) { } } -// Check if a key exists in a MapSlice and return its index and value -func findHelmValuesKey(m yaml.MapSlice, key string) (int, bool) { - for i, item := range m { - if item.Key == key { - return i, true +// Check if a key exists in a MappingNode and return the index of its value +func findHelmValuesKey(m *yaml.Node, key string) (int, bool) { + for i, item := range m.Content { + if i%2 == 0 && item.Value == key { + return i + 1, true } } return -1, false } +func nodeKindString(k yaml.Kind) string { + return map[yaml.Kind]string{ + yaml.DocumentNode: "DocumentNode", + yaml.SequenceNode: "SequenceNode", + yaml.MappingNode: "MappingNode", + yaml.ScalarNode: "ScalarNode", + yaml.AliasNode: "AliasNode", + }[k] +} + // set value of the parameter passed from the annotations. -func setHelmValue(currentValues *yaml.MapSlice, key string, value interface{}) error { +func setHelmValue(currentValues *yaml.Node, key string, value interface{}) error { + current := currentValues + + // an unmarshalled document has a DocumentNode at the root, but + // we navigate from a MappingNode. + if current.Kind == yaml.DocumentNode { + current = current.Content[0] + } + + if current.Kind != yaml.MappingNode { + return fmt.Errorf("unexpected type %s for root", nodeKindString(current.Kind)) + } + // Check if the full key exists - if idx, found := findHelmValuesKey(*currentValues, key); found { - (*currentValues)[idx].Value = value + if idx, found := findHelmValuesKey(current, key); found { + (*current).Content[idx].Value = value.(string) return nil } var err error keys := strings.Split(key, ".") - current := currentValues - var parent *yaml.MapSlice - parentIdx := -1 for i, k := range keys { - if idx, found := findHelmValuesKey(*current, k); found { + if idx, found := findHelmValuesKey(current, k); found { + // Navigate deeper into the map + current = (*current).Content[idx] + // unpack one level of alias; an alias of an alias is not supported + if current.Kind == yaml.AliasNode { + current = current.Alias + } if i == len(keys)-1 { // If we're at the final key, set the value and return - (*current)[idx].Value = value - return nil - } else { - // Navigate deeper into the map - if nestedMap, ok := (*current)[idx].Value.(yaml.MapSlice); ok { - parent = current - parentIdx = idx - current = &nestedMap + if current.Kind == yaml.ScalarNode { + current.Value = value.(string) + current.Tag = "!!str" } else { - return fmt.Errorf("unexpected type %T for key %s", (*current)[idx].Value, k) + return fmt.Errorf("unexpected type %s for key %s", nodeKindString(current.Kind), k) } + return nil + } else if current.Kind != yaml.MappingNode { + return fmt.Errorf("unexpected type %s for key %s", nodeKindString(current.Kind), k) } } else { - newCurrent := yaml.MapSlice{} - var newParent yaml.MapSlice - if i == len(keys)-1 { - newParent = append(*current, yaml.MapItem{Key: k, Value: value}) - } else { - newParent = append(*current, yaml.MapItem{Key: k, Value: newCurrent}) - } - - if parent == nil { - *currentValues = newParent + current.Content = append(current.Content, + &yaml.Node{ + Kind: yaml.ScalarNode, + Value: k, + Tag: "!!str", + }, + &yaml.Node{ + Kind: yaml.ScalarNode, + Value: value.(string), + Tag: "!!str", + }, + ) + return nil } else { - // if parentIdx has not been set (parent element is also new), set it to the last element - if parentIdx == -1 { - parentIdx = len(*parent) - 1 - if parentIdx < 0 { - parentIdx = 0 - } - } - (*parent)[parentIdx].Value = newParent + current.Content = append(current.Content, + &yaml.Node{ + Kind: yaml.ScalarNode, + Value: k, + Tag: "!!str", + }, + &yaml.Node{ + Kind: yaml.MappingNode, + Content: []*yaml.Node{}, + }, + ) + current = current.Content[len(current.Content)-1] } - - parent = &newParent - current = &newCurrent - parentIdx = -1 } } diff --git a/pkg/argocd/update_test.go b/pkg/argocd/update_test.go index 9dbeadae..3dca1869 100644 --- a/pkg/argocd/update_test.go +++ b/pkg/argocd/update_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/argoproj-labs/argocd-image-updater/ext/git" gitmock "github.com/argoproj-labs/argocd-image-updater/ext/git/mocks" @@ -1180,9 +1180,9 @@ func Test_MarshalParamsOverride(t *testing.T) { expected := ` kustomize: images: - - baz - - foo - - bar + - baz + - foo + - bar ` app := v1alpha1.Application{ ObjectMeta: v1.ObjectMeta{ @@ -1211,7 +1211,7 @@ kustomize: originalData := []byte(` kustomize: images: - - baz + - baz `) yaml, err := marshalParamsOverride(&app, originalData) require.NoError(t, err) @@ -1223,9 +1223,9 @@ kustomize: expected := ` kustomize: images: - - existing:latest - - updated:latest - - new + - existing:latest + - updated:latest + - new ` app := v1alpha1.Application{ ObjectMeta: v1.ObjectMeta{ @@ -1253,8 +1253,8 @@ kustomize: originalData := []byte(` kustomize: images: - - existing:latest - - updated:old + - existing:latest + - updated:old `) yaml, err := marshalParamsOverride(&app, originalData) require.NoError(t, err) @@ -1292,15 +1292,15 @@ kustomize: expected := ` helm: parameters: - - name: baz - value: baz - forcestring: false - - name: foo - value: bar - forcestring: true - - name: bar - value: foo - forcestring: true + - name: baz + value: baz + forcestring: false + - name: foo + value: bar + forcestring: true + - name: bar + value: foo + forcestring: true ` app := v1alpha1.Application{ ObjectMeta: v1.ObjectMeta{ @@ -1352,12 +1352,12 @@ helm: expected := ` helm: parameters: - - name: foo - value: bar - forcestring: true - - name: bar - value: foo - forcestring: true + - name: foo + value: bar + forcestring: true + - name: bar + value: foo + forcestring: true ` app := v1alpha1.Application{ ObjectMeta: v1.ObjectMeta{ @@ -1403,12 +1403,12 @@ helm: expected := ` helm: parameters: - - name: foo - value: bar - forcestring: true - - name: bar - value: foo - forcestring: true + - name: foo + value: bar + forcestring: true + - name: bar + value: foo + forcestring: true ` app := v1alpha1.Application{ ObjectMeta: v1.ObjectMeta{ @@ -2199,122 +2199,307 @@ replicas: 1 func Test_SetHelmValue(t *testing.T) { t.Run("Update existing Key", func(t *testing.T) { - expected := yaml.MapSlice{ - {Key: "image", Value: yaml.MapSlice{ - {Key: "attributes", Value: yaml.MapSlice{ - {Key: "name", Value: "repo-name"}, - {Key: "tag", Value: "v2.0.0"}, - }}, - }}, - } - - input := yaml.MapSlice{ - {Key: "image", Value: yaml.MapSlice{ - {Key: "attributes", Value: yaml.MapSlice{ - {Key: "name", Value: "repo-name"}, - {Key: "tag", Value: "v1.0.0"}, - }}, - }}, - } + expected := ` +image: + attributes: + name: repo-name + tag: v2.0.0 +` + + inputData := []byte(` +image: + attributes: + name: repo-name + tag: v1.0.0 +`) + input := yaml.Node{} + err := yaml.Unmarshal(inputData, &input) + require.NoError(t, err) + key := "image.attributes.tag" value := "v2.0.0" - err := setHelmValue(&input, key, value) + err = setHelmValue(&input, key, value) + require.NoError(t, err) + + output, err := marshalWithIndent(&input, 2) require.NoError(t, err) - assert.Equal(t, expected, input) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(output))) }) t.Run("Update Key with dots", func(t *testing.T) { - expected := yaml.MapSlice{ - {Key: "image.attributes.tag", Value: "v2.0.0"}, - } + expected := `image.attributes.tag: v2.0.0` + + inputData := []byte(`image.attributes.tag: v1.0.0`) + input := yaml.Node{} + err := yaml.Unmarshal(inputData, &input) + require.NoError(t, err) - input := yaml.MapSlice{ - {Key: "image.attributes.tag", Value: "v1.0.0"}, - } key := "image.attributes.tag" value := "v2.0.0" - err := setHelmValue(&input, key, value) + err = setHelmValue(&input, key, value) + require.NoError(t, err) + + output, err := marshalWithIndent(&input, 2) require.NoError(t, err) - assert.Equal(t, expected, input) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(output))) }) t.Run("Key not found", func(t *testing.T) { - expected := yaml.MapSlice{ - {Key: "image", Value: yaml.MapSlice{ - {Key: "attributes", Value: yaml.MapSlice{ - {Key: "name", Value: "repo-name"}, - {Key: "tag", Value: "v2.0.0"}, - }}, - }}, - } + expected := ` +image: + attributes: + name: repo-name + tag: v2.0.0 +` - input := yaml.MapSlice{ - {Key: "image", Value: yaml.MapSlice{ - {Key: "attributes", Value: yaml.MapSlice{ - {Key: "name", Value: "repo-name"}, - }}, - }}, - } + inputData := []byte(` +image: + attributes: + name: repo-name +`) + input := yaml.Node{} + err := yaml.Unmarshal(inputData, &input) + require.NoError(t, err) key := "image.attributes.tag" value := "v2.0.0" - err := setHelmValue(&input, key, value) + err = setHelmValue(&input, key, value) + require.NoError(t, err) + + output, err := marshalWithIndent(&input, 2) require.NoError(t, err) - assert.Equal(t, expected, input) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(output))) }) t.Run("Root key not found", func(t *testing.T) { - expected := yaml.MapSlice{ - {Key: "name", Value: "repo-name"}, - {Key: "tag", Value: "v2.0.0"}, - } + expected := ` +name: repo-name +tag: v2.0.0 +` - input := yaml.MapSlice{ - {Key: "name", Value: "repo-name"}, - } + inputData := []byte(`name: repo-name`) + input := yaml.Node{} + err := yaml.Unmarshal(inputData, &input) + require.NoError(t, err) key := "tag" value := "v2.0.0" - err := setHelmValue(&input, key, value) + err = setHelmValue(&input, key, value) require.NoError(t, err) - assert.Equal(t, expected, input) + + output, err := marshalWithIndent(&input, 2) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(output))) }) t.Run("Empty values with deep key", func(t *testing.T) { - expected := yaml.MapSlice{ - {Key: "image", Value: yaml.MapSlice{ - {Key: "attributes", Value: yaml.MapSlice{ - {Key: "tag", Value: "v2.0.0"}, - }}, - }}, - } + // this uses inline syntax because the input data + // needed is an empty map, which can only be expressed as {}. + expected := `{image: {attributes: {tag: v2.0.0}}}` - input := yaml.MapSlice{} + inputData := []byte(`{}`) + input := yaml.Node{} + err := yaml.Unmarshal(inputData, &input) + require.NoError(t, err) key := "image.attributes.tag" value := "v2.0.0" - err := setHelmValue(&input, key, value) + err = setHelmValue(&input, key, value) require.NoError(t, err) - assert.Equal(t, expected, input) + + output, err := marshalWithIndent(&input, 2) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(output))) }) t.Run("Unexpected type for key", func(t *testing.T) { - input := yaml.MapSlice{ - {Key: "image", Value: yaml.MapSlice{ - {Key: "attributes", Value: "v1.0.0"}, - }}, - } + inputData := []byte(` +image: + attributes: v1.0.0 +`) + input := yaml.Node{} + err := yaml.Unmarshal(inputData, &input) + require.NoError(t, err) + key := "image.attributes.tag" value := "v2.0.0" - err := setHelmValue(&input, key, value) + err = setHelmValue(&input, key, value) assert.Error(t, err) - assert.Equal(t, "unexpected type string for key attributes", err.Error()) + assert.Equal(t, "unexpected type ScalarNode for key attributes", err.Error()) + }) + + t.Run("Aliases, comments, and multiline strings are preserved", func(t *testing.T) { + expected := ` +image: + attributes: + name: &repo repo-name + tag: v2.0.0 + # this is a comment + multiline: | + one + two + three + alias: *repo +` + + inputData := []byte(` +image: + attributes: + name: &repo repo-name + tag: v1.0.0 + # this is a comment + multiline: | + one + two + three + alias: *repo +`) + input := yaml.Node{} + err := yaml.Unmarshal(inputData, &input) + require.NoError(t, err) + + key := "image.attributes.tag" + value := "v2.0.0" + + err = setHelmValue(&input, key, value) + require.NoError(t, err) + + output, err := marshalWithIndent(&input, 2) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(output))) + }) + + t.Run("Aliases to mappings are followed", func(t *testing.T) { + expected := ` +global: + attributes: &attrs + name: &repo repo-name + tag: v2.0.0 +image: + attributes: *attrs +` + + inputData := []byte(` +global: + attributes: &attrs + name: &repo repo-name + tag: v1.0.0 +image: + attributes: *attrs +`) + input := yaml.Node{} + err := yaml.Unmarshal(inputData, &input) + require.NoError(t, err) + + key := "image.attributes.tag" + value := "v2.0.0" + + err = setHelmValue(&input, key, value) + require.NoError(t, err) + + output, err := marshalWithIndent(&input, 2) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(output))) + }) + + t.Run("Aliases to scalars are followed", func(t *testing.T) { + expected := ` +image: + attributes: + name: repo-name + version: &ver v2.0.0 + tag: *ver +` + + inputData := []byte(` +image: + attributes: + name: repo-name + version: &ver v1.0.0 + tag: *ver +`) + input := yaml.Node{} + err := yaml.Unmarshal(inputData, &input) + require.NoError(t, err) + + key := "image.attributes.tag" + value := "v2.0.0" + + err = setHelmValue(&input, key, value) + require.NoError(t, err) + + output, err := marshalWithIndent(&input, 2) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(output))) + }) + + t.Run("Indentation is guessed from nested mappings", func(t *testing.T) { + expected := ` +unguessable: + - unguessable +image: + attributes: + tag: v2.0.0 +` + + inputData := []byte(` +unguessable: +- unguessable +image: + attributes: + tag: v1.0.0 +`) + input := yaml.Node{} + err := yaml.Unmarshal(inputData, &input) + require.NoError(t, err) + indent := guessIndent(&input) + + key := "image.attributes.tag" + value := "v2.0.0" + + err = setHelmValue(&input, key, value) + require.NoError(t, err) + + output, err := marshalWithIndent(&input, indent) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(output))) + }) + + t.Run("Indentation is guessed from indented lists", func(t *testing.T) { + expected := ` +unguessable: [unguessable] +guessable: + - guessable +image: + attributes: + tag: v2.0.0 +` + + inputData := []byte(` +unguessable: [unguessable] +guessable: + - guessable +`) + input := yaml.Node{} + err := yaml.Unmarshal(inputData, &input) + require.NoError(t, err) + indent := guessIndent(&input) + + key := "image.attributes.tag" + value := "v2.0.0" + + err = setHelmValue(&input, key, value) + require.NoError(t, err) + + output, err := marshalWithIndent(&input, indent) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(output))) }) }