Skip to content

add --image-url-substitution #296

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@ enrich version checking on image tags:
used to change the URL for where to lookup where the latest image version
is. In this example, the current version of `my-container` will be compared
against the image versions in the `docker.io/bitnami/etcd` registry.
(This also overrrides any substitution/replacement from `--image-url-substitution`.)

## Known configurations

Expand Down
20 changes: 13 additions & 7 deletions cmd/app/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,17 @@ package app
import (
"context"
"fmt"
"os"

"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"k8s.io/client-go/kubernetes"
_ "k8s.io/client-go/plugin/pkg/client/auth" // Load all auth plugins
"os"

"github.com/jetstack/version-checker/pkg/api"
"github.com/jetstack/version-checker/pkg/client"
"github.com/jetstack/version-checker/pkg/controller"
"github.com/jetstack/version-checker/pkg/controller/checker"
"github.com/jetstack/version-checker/pkg/metrics"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
_ "k8s.io/client-go/plugin/pkg/client/auth" // Load all auth plugins
)

const (
Expand Down Expand Up @@ -74,8 +74,14 @@ func NewCommand(ctx context.Context) *cobra.Command {

log.Infof("flag --test-all-containers=%t %s", opts.DefaultTestAll, defaultTestAllInfoMsg)

c := controller.New(opts.CacheTimeout, metrics,
client, kubeClient, log, opts.DefaultTestAll)
var imageURLSubstitution *checker.Substitution
if opts.ImageURLSubstitution != "" {
if imageURLSubstitution, err = checker.NewSubstitutionFromSedCommand(opts.ImageURLSubstitution); err != nil {
return fmt.Errorf("failed to parse --image-url-substitution %q: %s", opts.ImageURLSubstitution, err)
}
}

c := controller.New(opts.CacheTimeout, metrics, client, kubeClient, log, opts.DefaultTestAll, imageURLSubstitution)

return c.Run(ctx, opts.CacheTimeout/2)
},
Expand Down
9 changes: 9 additions & 0 deletions cmd/app/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ type Options struct {
MetricsServingAddress string
DefaultTestAll bool
CacheTimeout time.Duration
ImageURLSubstitution string
LogLevel string

kubeConfigFlags *genericclioptions.ConfigFlags
Expand Down Expand Up @@ -114,6 +115,14 @@ func (o *Options) addAppFlags(fs *pflag.FlagSet) {
"The time for an image version in the cache to be considered fresh. Images "+
"will be rechecked after this interval.")

fs.StringVarP(&o.ImageURLSubstitution,
"image-url-substitution", "", "",
"Image URL substitution. In case you want to apply a replacement to all image URLs. "+
"E.g. when you mirrored them and want to check the originals. "+
"The format follows the sed substitution command syntax. "+
"E.g. s#myacr.azurecr.io/mirror/##g to remove your registry from: myacr.azurecr.io/mirror/docker.io/alpine:3"+
"(Any override-url.version-checker.io/my-container annotation would still override everything afterwards.)")

fs.StringVarP(&o.LogLevel,
"log-level", "v", "info",
"Log level (debug, info, warn, error, fatal, panic).")
Expand Down
21 changes: 18 additions & 3 deletions pkg/controller/checker/checker.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ import (
)

type Checker struct {
search search.Searcher
search search.Searcher
imageURLSubstitution *Substitution
}

type Result struct {
Expand All @@ -24,9 +25,10 @@ type Result struct {
ImageURL string
}

func New(search search.Searcher) *Checker {
func New(search search.Searcher, imageURLSubstitution *Substitution) *Checker {
return &Checker{
search: search,
search: search,
imageURLSubstitution: imageURLSubstitution,
}
}

Expand All @@ -46,6 +48,7 @@ func (c *Checker) Container(ctx context.Context, log *logrus.Entry, pod *corev1.
usingTag = false
}

imageURL = c.substituteImageURL(log, imageURL)
imageURL = c.overrideImageURL(log, imageURL, opts)

if opts.UseSHA {
Expand All @@ -60,6 +63,18 @@ func (c *Checker) handleLatestOrEmptyTag(log *logrus.Entry, currentTag, currentS
log.WithField("module", "checker").Debugf("image using %q tag, comparing image SHA %q", currentTag, currentSHA)
}

func (c *Checker) substituteImageURL(log *logrus.Entry, imageURL string) string {

if c.imageURLSubstitution == nil {
return imageURL
}
newImageURL := c.imageURLSubstitution.Pattern.ReplaceAllString(imageURL, c.imageURLSubstitution.Substitute)
if newImageURL != imageURL {
log.Debugf("substituting image URL %s -> %s", imageURL, newImageURL)
}
return newImageURL
}

func (c *Checker) overrideImageURL(log *logrus.Entry, imageURL string, opts *api.Options) string {
if opts.OverrideURL != nil && *opts.OverrideURL != imageURL {
log.Debugf("overriding image URL %s -> %s", imageURL, *opts.OverrideURL)
Expand Down
42 changes: 33 additions & 9 deletions pkg/controller/checker/checker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package checker

import (
"context"
"github.com/stretchr/testify/require"
"reflect"
"testing"

Expand All @@ -15,11 +16,12 @@ import (

func TestContainer(t *testing.T) {
tests := map[string]struct {
statusSHA string
imageURL string
opts *api.Options
searchResp *api.ImageTag
expResult *Result
statusSHA string
imageURL string
opts *api.Options
imageURLSubstitution string
searchResp *api.ImageTag
expResult *Result
}{
"no status sha should return nil, nil": {
statusSHA: "",
Expand Down Expand Up @@ -165,6 +167,22 @@ func TestContainer(t *testing.T) {
IsLatest: true,
},
},
"if latest is latest version, and imageURLSubstitution is set, then return true": {
statusSHA: "myregistry.example.com/mirrored/quay.io/jetstack/version-checker:latest@sha:123",
imageURL: "myregistry.example.com/mirrored/quay.io/jetstack/version-checker:latest",
opts: new(api.Options),
imageURLSubstitution: "s/myregistry.example.com\\/mirrored\\///g",
searchResp: &api.ImageTag{
Tag: "",
SHA: "sha:123",
},
expResult: &Result{
CurrentVersion: "sha:123",
LatestVersion: "sha:123",
ImageURL: "quay.io/jetstack/version-checker",
IsLatest: true,
},
},
"if using v0.2.0 with use sha, but not latest, return false": {
statusSHA: "localhost:5000/version-checker@sha:123",
imageURL: "localhost:5000/version-checker:v0.2.0",
Expand Down Expand Up @@ -263,7 +281,13 @@ func TestContainer(t *testing.T) {

for name, test := range tests {
t.Run(name, func(t *testing.T) {
checker := New(search.New().With(test.searchResp, nil))
var err error
var imageURLSubstitution *Substitution
if test.imageURLSubstitution != "" {
imageURLSubstitution, err = NewSubstitutionFromSedCommand(test.imageURLSubstitution)
require.NoError(t, err)
}
checker := New(search.New().With(test.searchResp, nil), imageURLSubstitution)
pod := &corev1.Pod{
Status: corev1.PodStatus{
ContainerStatuses: []corev1.ContainerStatus{
Expand Down Expand Up @@ -398,7 +422,7 @@ func TestIsLatestOrEmptyTag(t *testing.T) {

for name, test := range tests {
t.Run(name, func(t *testing.T) {
checker := New(search.New())
checker := New(search.New(), nil)
if is := checker.isLatestOrEmptyTag(test.tag); is != test.expIs {
t.Errorf("unexpected isLatestOrEmptyTag exp=%t got=%t",
test.expIs, is)
Expand Down Expand Up @@ -475,7 +499,7 @@ func TestIsLatestSemver(t *testing.T) {

for name, test := range tests {
t.Run(name, func(t *testing.T) {
checker := New(search.New().With(test.searchResp, nil))
checker := New(search.New().With(test.searchResp, nil), nil)
latestImage, isLatest, err := checker.isLatestSemver(context.TODO(), test.imageURL, test.currentSHA, test.currentImage, nil)
if err != nil {
t.Fatal(err)
Expand Down Expand Up @@ -530,7 +554,7 @@ func TestIsLatestSHA(t *testing.T) {

for name, test := range tests {
t.Run(name, func(t *testing.T) {
checker := New(search.New().With(test.searchResp, nil))
checker := New(search.New().With(test.searchResp, nil), nil)
result, err := checker.isLatestSHA(context.TODO(), test.imageURL, test.currentSHA, nil)
if err != nil {
t.Fatal(err)
Expand Down
59 changes: 59 additions & 0 deletions pkg/controller/checker/substitution.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package checker

import (
"fmt"
"regexp"
"strings"
)

type Substitution struct {
Pattern *regexp.Regexp
Substitute string
All bool
}

func NewSubstitutionFromSedCommand(sedCommand string) (*Substitution, error) {
pattern, substitute, flags, err := splitSedSubstitutionCommand(sedCommand)
if err != nil {
return nil, err
}
compiledPattern, err := regexp.Compile(pattern)
if err != nil {
return nil, fmt.Errorf("sed command for substitution has regex that does not compile: %s %w", pattern, err)
}
var all bool
if flags == "g" {
all = true
} else if flags != "" {
return nil, fmt.Errorf("sed command for substitution only supports the 'g' flag: %s", flags)
}

return &Substitution{
Pattern: compiledPattern,
Substitute: substitute,
All: all,
}, nil
}

func splitSedSubstitutionCommand(sedCommand string) (string, string, string, error) {
if len(sedCommand) < 4 {
return "", "", "", fmt.Errorf("sed command for substitution seems to short: %s", sedCommand)
}
if sedCommand[0] != 's' {
return "", "", "", fmt.Errorf("sed command for substitution should start with s: %s", sedCommand)
}
separator := regexp.QuoteMeta(sedCommand[1:2])
group := fmt.Sprintf(`((?:.|\\[%s])*)`, separator)
pattern := `^s` + separator + group + separator + group + separator + group + `$`
matcher, err := regexp.Compile(pattern)
if err != nil {
return "", "", "", fmt.Errorf("regexp to parse sed command for substitution does not compile: %s %w", pattern, err)
}
submatches := matcher.FindStringSubmatch(sedCommand)
if len(submatches) != 4 {
return "", "", "", fmt.Errorf("sed command for substitution could not be parsed: %s", pattern)
}
return strings.ReplaceAll(submatches[1], "\\"+separator, separator),
strings.ReplaceAll(submatches[2], "\\"+separator, separator),
submatches[3], nil
}
Comment on lines +39 to +59
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there any reason why you chose SED over a standard/traditional Regex? (which looks like you're doing anyway after all the sed-specific searches)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I used the sed-like format/syntax to have the search and replacement string in one cli option. It seemed like a well-known/popular/widely-used format to me (s/search/replace). For the rest it doesn't have anything to do with sed.

4 changes: 3 additions & 1 deletion pkg/controller/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,21 +51,23 @@ func New(
kubeClient kubernetes.Interface,
log *logrus.Entry,
defaultTestAll bool,
imageURLSubstitution *checker.Substitution,
) *Controller {
workqueue := workqueue.NewTypedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[any]())
scheduledWorkQueue := scheduler.NewScheduledWorkQueue(clock.RealClock{}, workqueue.Add)

log = log.WithField("module", "controller")
versionGetter := version.New(log, imageClient, cacheTimeout)
search := search.New(log, cacheTimeout, versionGetter)
checker := checker.New(search, imageURLSubstitution)

c := &Controller{
log: log,
kubeClient: kubeClient,
workqueue: workqueue,
scheduledWorkQueue: scheduledWorkQueue,
metrics: metrics,
checker: checker.New(search),
checker: checker,
defaultTestAll: defaultTestAll,
}

Expand Down
10 changes: 5 additions & 5 deletions pkg/controller/controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ func TestNewController(t *testing.T) {
metrics := &metrics.Metrics{}
imageClient := &client.Client{}

controller := New(5*time.Minute, metrics, imageClient, kubeClient, testLogger, true)
controller := New(5*time.Minute, metrics, imageClient, kubeClient, testLogger, true, nil)

assert.NotNil(t, controller)
assert.Equal(t, controller.defaultTestAll, true)
Expand All @@ -43,7 +43,7 @@ func TestRun(t *testing.T) {
kubeClient := fake.NewSimpleClientset()
metrics := &metrics.Metrics{}
imageClient := &client.Client{}
controller := New(5*time.Minute, metrics, imageClient, kubeClient, testLogger, true)
controller := New(5*time.Minute, metrics, imageClient, kubeClient, testLogger, true, nil)

ctx, cancel := context.WithCancel(context.Background())

Expand Down Expand Up @@ -72,7 +72,7 @@ func TestAddObject(t *testing.T) {
kubeClient := fake.NewSimpleClientset()
metrics := &metrics.Metrics{}
imageClient := &client.Client{}
controller := New(5*time.Minute, metrics, imageClient, kubeClient, testLogger, true)
controller := New(5*time.Minute, metrics, imageClient, kubeClient, testLogger, true, nil)

obj := &corev1.Pod{}
controller.addObject(obj)
Expand All @@ -99,7 +99,7 @@ func TestDeleteObject(t *testing.T) {
kubeClient := fake.NewSimpleClientset()
metrics := &metrics.Metrics{}
imageClient := &client.Client{}
controller := New(5*time.Minute, metrics, imageClient, kubeClient, testLogger, true)
controller := New(5*time.Minute, metrics, imageClient, kubeClient, testLogger, true, nil)

pod := &corev1.Pod{
Spec: corev1.PodSpec{
Expand All @@ -120,7 +120,7 @@ func TestProcessNextWorkItem(t *testing.T) {
kubeClient := fake.NewSimpleClientset()
metrics := &metrics.Metrics{}
imageClient := &client.Client{}
controller := New(5*time.Minute, metrics, imageClient, kubeClient, testLogger, true)
controller := New(5*time.Minute, metrics, imageClient, kubeClient, testLogger, true, nil)

ctx, cancel := context.WithCancel(context.Background())
defer cancel()
Expand Down
8 changes: 4 additions & 4 deletions pkg/controller/sync_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ func TestController_Sync(t *testing.T) {
metrics := &metrics.Metrics{}
imageClient := &client.Client{}
searcher := search.New(log, 5*time.Minute, version.New(log, imageClient, 5*time.Minute))
checker := checker.New(searcher)
checker := checker.New(searcher, nil)

controller := &Controller{
log: log,
Expand Down Expand Up @@ -59,7 +59,7 @@ func TestController_SyncContainer(t *testing.T) {
metrics := &metrics.Metrics{}
imageClient := &client.Client{}
searcher := search.New(log, 5*time.Minute, version.New(log, imageClient, 5*time.Minute))
checker := checker.New(searcher)
checker := checker.New(searcher, nil)

controller := &Controller{
log: log,
Expand Down Expand Up @@ -90,7 +90,7 @@ func TestController_CheckContainer(t *testing.T) {
metrics := &metrics.Metrics{}
imageClient := &client.Client{}
searcher := search.New(log, 5*time.Minute, version.New(log, imageClient, 5*time.Minute))
checker := checker.New(searcher)
checker := checker.New(searcher, nil)

controller := &Controller{
log: log,
Expand Down Expand Up @@ -118,7 +118,7 @@ func TestController_SyncContainer_NoVersionFound(t *testing.T) {
metrics := &metrics.Metrics{}
imageClient := &client.Client{}
searcher := search.New(log, 5*time.Minute, version.New(log, imageClient, 5*time.Minute))
checker := checker.New(searcher)
checker := checker.New(searcher, nil)

controller := &Controller{
log: log,
Expand Down