From 7712ec324fef93f3b6409e46826b75c24a3481ce Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 28 Apr 2025 16:01:35 +0100 Subject: [PATCH 1/7] Consider all SHA's of a reference, including manifests --- .github/workflows/build-test.yaml | 15 +- .github/workflows/helm-docs.yaml | 4 +- .github/workflows/helm-test.yaml | 28 ++- .github/workflows/release.yaml | 29 ++- cmd/app/app.go | 3 +- cmd/app/options.go | 112 ++++++---- cmd/app/options_test.go | 20 +- go.mod | 15 +- go.sum | 31 ++- pkg/api/options.go | 16 +- pkg/api/types.go | 24 +++ pkg/client/acr/acr.go | 205 +++---------------- pkg/client/acr/auth.go | 163 +++++++++++++++ pkg/client/acr/types.go | 30 +++ pkg/client/client.go | 10 +- pkg/client/client_test.go | 18 +- pkg/client/docker/docker.go | 52 ++++- pkg/client/docker/docker_test.go | 138 +++++++++++++ pkg/client/docker/types.go | 11 +- pkg/client/ecr/ecr.go | 33 +-- pkg/client/fallback/fallback.go | 6 +- pkg/client/gcr/gcr.go | 43 +++- pkg/client/gcr/path.go | 14 +- pkg/client/gcr/path_test.go | 19 +- pkg/client/ghcr/ghcr.go | 44 ++-- pkg/client/ghcr/ghcr_test.go | 9 +- pkg/client/oci/helpers.go | 25 +++ pkg/client/oci/helpers_test.go | 77 +++++++ pkg/client/oci/oci.go | 112 +++++++++- pkg/client/oci/oci_test.go | 106 +++++----- pkg/client/quay/api_types.go | 35 ++++ pkg/client/quay/pager.go | 26 ++- pkg/client/quay/path_test.go | 19 +- pkg/client/quay/quay.go | 77 +++---- pkg/client/selfhosted/api_types.go | 50 ++++- pkg/client/selfhosted/path.go | 3 + pkg/client/selfhosted/path_test.go | 25 ++- pkg/client/selfhosted/selfhosted.go | 64 +++--- pkg/client/selfhosted/selfhosted_test.go | 96 ++++++++- pkg/client/util/http_backoff.go | 19 ++ pkg/client/util/http_backoff_limiter.go | 49 +++++ pkg/client/util/http_backoff_limiter_test.go | 65 ++++++ pkg/client/util/util.go | 32 ++- pkg/client/util/util_test.go | 90 ++++++++ pkg/controller/checker/checker.go | 47 ++++- pkg/controller/checker/checker_test.go | 122 ++++++++++- pkg/controller/options/options.go | 2 + pkg/controller/pod_controller.go | 48 ++++- pkg/controller/pod_controller_test.go | 9 +- pkg/controller/search/search.go | 6 +- pkg/version/filters.go | 4 + pkg/version/helpers.go | 55 +++++ pkg/version/semver/semver.go | 6 +- pkg/version/version.go | 65 +----- 54 files changed, 1796 insertions(+), 630 deletions(-) create mode 100644 pkg/client/acr/auth.go create mode 100644 pkg/client/acr/types.go create mode 100644 pkg/client/docker/docker_test.go create mode 100644 pkg/client/oci/helpers.go create mode 100644 pkg/client/oci/helpers_test.go create mode 100644 pkg/client/quay/api_types.go create mode 100644 pkg/client/util/http_backoff.go create mode 100644 pkg/client/util/http_backoff_limiter.go create mode 100644 pkg/client/util/http_backoff_limiter_test.go create mode 100644 pkg/version/helpers.go diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index b2b3bd23..7193ba4c 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -19,10 +19,12 @@ jobs: steps: - name: Checkout code uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Setup Golang uses: actions/setup-go@v5 with: go-version-file: go.mod + - name: Run golangci-lint uses: golangci/golangci-lint-action@1481404843c368bc19ca9406f87d6e0fc97bdcfd # v7.0.0 with: @@ -34,6 +36,10 @@ jobs: runs-on: ubuntu-latest name: Run govulncheck steps: + # We only need to checkout as govuln does the go setup... + - name: Checkout code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - id: govulncheck uses: golang/govulncheck-action@v1 with: @@ -45,8 +51,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 # v3.5.3 - - name: Setup Go + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Setup Golang uses: actions/setup-go@v5 with: go-version-file: go.mod @@ -88,6 +95,7 @@ jobs: steps: - name: Checkout code uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -106,6 +114,9 @@ jobs: tags: quay.io/jetstack/version-checker:${{github.sha}} cache-from: type=gha cache-to: type=gha,mode=max + attests: |- + type=sbom,generator=image + type=provenance,mode=max - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@0.30.0 diff --git a/.github/workflows/helm-docs.yaml b/.github/workflows/helm-docs.yaml index 3af87f55..d53def80 100644 --- a/.github/workflows/helm-docs.yaml +++ b/.github/workflows/helm-docs.yaml @@ -4,8 +4,8 @@ on: workflow_call: push: paths: - - '!*.md' - - 'deploy/charts/version-checker/**' + - "!*.md" + - "deploy/charts/version-checker/**" branches: - main diff --git a/.github/workflows/helm-test.yaml b/.github/workflows/helm-test.yaml index 43dd2089..a5f9b8c9 100644 --- a/.github/workflows/helm-test.yaml +++ b/.github/workflows/helm-test.yaml @@ -2,10 +2,10 @@ name: Test Helm Chart on: pull_request: paths: - - '!*.md' - - 'deploy/charts/version-checker/**' + - "!*.md" + - "deploy/charts/version-checker/**" branches: - - 'main' + - "main" concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -14,14 +14,19 @@ concurrency: jobs: lint: permissions: - contents: read # for actions/checkout to fetch code - pull-requests: read # for golangci/golangci-lint-action to fetch pull requests + contents: read # for actions/checkout to fetch code + pull-requests: read # for golangci/golangci-lint-action to fetch pull requests name: Lint Helm Chart runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Setup Golang + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + - uses: azure/setup-helm@v4 - run: helm lint deploy/charts/version-checker @@ -33,9 +38,12 @@ jobs: - name: Checkout code uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - uses: azure/setup-helm@v4 + - name: Setup Golang + uses: actions/setup-go@v5 with: - token: ${{ github.token }} + go-version-file: go.mod + + - uses: azure/setup-helm@v4 - name: Install helm Plugins run: | @@ -57,7 +65,9 @@ jobs: steps: - name: Checkout code uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Install Kyverno CLI uses: kyverno/action-install-cli@v0.2.0 - - uses: azure/setup-helm@v4 - - run: kyverno apply -p https://github.com/kyverno/policies/pod-security/restricted --git-branch main --resource <(helm template deploy/charts/version-checker/) + + - run: |- + kyverno apply -p https://github.com/kyverno/policies/pod-security/restricted --git-branch main --resource <(helm template deploy/charts/version-checker/) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 6debc21b..df2a065a 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -15,15 +15,19 @@ jobs: prepare-release: # Don't push back to a tag! if: ${{ !startsWith(github.ref, 'refs/tags/') }} - name: Prepair release + name: Prepare release runs-on: ubuntu-latest permissions: pull-requests: write contents: write steps: - - uses: actions/checkout@v4 + - name: Checkout code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Setup Golang + uses: actions/setup-go@v5 with: - fetch-depth: 0 + go-version-file: go.mod + - uses: bhowell2/github-substring-action@1.0.2 id: release_number with: @@ -113,10 +117,8 @@ jobs: helm-release: runs-on: ubuntu-latest steps: - # Checkout our Repo - - uses: actions/checkout@v4 - with: - path: version-checker + - name: Checkout code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: checkout jetstack-charts uses: actions/checkout@v4 @@ -134,6 +136,9 @@ jobs: run: | helm package version-checker/deploy/charts/version-checker -d jetstack-charts/charts/ + - name: Login to Quay.io + run: echo "${{ secrets.QUAY_ROBOT_TOKEN }}" | helm registry login quay.io -u ${{ secrets.QUAY_USERNAME }} --password-stdin + - name: Creating PR if: startsWith(github.ref, 'refs/tags/') uses: peter-evans/create-pull-request@v7 @@ -149,10 +154,16 @@ jobs: base: main draft: ${{ !startsWith(github.ref, 'refs/tags/') }} + - name: Push to Quay + run: |- + helm push jetstack-charts/charts/version-checker-${{ github.ref_name }}.tgz oci://quay.io/quay.io/jetstack/version-checker/chart + docker-release: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - name: Checkout code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -185,8 +196,6 @@ jobs: contents: write runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - name: Create Release / Change Logs uses: softprops/action-gh-release@v2 with: diff --git a/cmd/app/app.go b/cmd/app/app.go index e18d85f0..bff6a403 100644 --- a/cmd/app/app.go +++ b/cmd/app/app.go @@ -59,7 +59,7 @@ func NewCommand(ctx context.Context) *cobra.Command { return fmt.Errorf("failed to build kubernetes rest config: %s", err) } - log.Infof("flag --test-all-containers=%t %s", opts.DefaultTestAll, defaultTestAllInfoMsg) + log.Warnf("flag --test-all-containers=%t %s", opts.DefaultTestAll, defaultTestAllInfoMsg) mgr, err := ctrl.NewManager(restConfig, ctrl.Options{ LeaderElection: false, @@ -115,6 +115,7 @@ func NewCommand(ctx context.Context) *cobra.Command { client, mgr.GetClient(), log, + opts.RequeueDuration, opts.DefaultTestAll, ) diff --git a/cmd/app/options.go b/cmd/app/options.go index 5359f3a7..94dff98e 100644 --- a/cmd/app/options.go +++ b/cmd/app/options.go @@ -64,19 +64,26 @@ var ( // Options is a struct to hold options for the version-checker. type Options struct { + kubeConfigFlags *genericclioptions.ConfigFlags + + Client client.Options MetricsServingAddress string - DefaultTestAll bool - CacheTimeout time.Duration LogLevel string - PprofBindAddress string + PprofBindAddress string + selfhosted selfhosted.Options + + CacheTimeout time.Duration + RequeueDuration time.Duration GracefulShutdownTimeout time.Duration CacheSyncPeriod time.Duration - kubeConfigFlags *genericclioptions.ConfigFlags - selfhosted selfhosted.Options + DefaultTestAll bool +} - Client client.Options +type envMatcher struct { + re *regexp.Regexp + action func(matches []string, value string) } func (o *Options) addFlags(cmd *cobra.Command) { @@ -124,6 +131,10 @@ func (o *Options) addAppFlags(fs *pflag.FlagSet) { "The time for an image version in the cache to be considered fresh. Images "+ "will be rechecked after this interval.") + fs.DurationVarP(&o.RequeueDuration, + "requeue-duration", "r", time.Hour, + "The time a pod will be re-checked for new versions/tags") + fs.StringVarP(&o.LogLevel, "log-level", "v", "info", "Log level (debug, info, warn, error, fatal, panic).") @@ -358,56 +369,81 @@ func (o *Options) assignSelfhosted(envs []string) { } initOptions := func(name string) { + if name == "" { + panic("Not meant to be empty!") + } if o.Client.Selfhosted[name] == nil { o.Client.Selfhosted[name] = new(selfhosted.Options) } } - regexActions := map[*regexp.Regexp]func(matches []string, value string){ - selfhostedHostReg: func(matches []string, value string) { - initOptions(matches[1]) - o.Client.Selfhosted[matches[1]].Host = value + // Go maps iterate in random order - Using a slice to consistency + regexActions := []envMatcher{ + { + re: selfhostedTokenPath, + action: func(matches []string, value string) { + initOptions(matches[1]) + o.Client.Selfhosted[matches[1]].TokenPath = value + }, }, - selfhostedUsernameReg: func(matches []string, value string) { - initOptions(matches[1]) - o.Client.Selfhosted[matches[1]].Username = value + { + re: selfhostedTokenReg, + action: func(matches []string, value string) { + initOptions(matches[1]) + o.Client.Selfhosted[matches[1]].Bearer = value + }, }, - selfhostedPasswordReg: func(matches []string, value string) { - initOptions(matches[1]) - o.Client.Selfhosted[matches[1]].Password = value + // All your other patterns (host, username, password, insecure, capath...) + { + re: selfhostedHostReg, + action: func(matches []string, value string) { + initOptions(matches[1]) + o.Client.Selfhosted[matches[1]].Host = value + }, }, - selfhostedTokenPath: func(matches []string, value string) { - initOptions(matches[1]) - o.Client.Selfhosted[matches[1]].TokenPath = value + { + re: selfhostedUsernameReg, + action: func(matches []string, value string) { + initOptions(matches[1]) + o.Client.Selfhosted[matches[1]].Username = value + }, }, - selfhostedTokenReg: func(matches []string, value string) { - initOptions(matches[1]) - o.Client.Selfhosted[matches[1]].Bearer = value + { + re: selfhostedPasswordReg, + action: func(matches []string, value string) { + initOptions(matches[1]) + o.Client.Selfhosted[matches[1]].Password = value + }, }, - selfhostedInsecureReg: func(matches []string, value string) { - initOptions(matches[1]) - if val, err := strconv.ParseBool(value); err == nil { - o.Client.Selfhosted[matches[1]].Insecure = val - } + { + re: selfhostedInsecureReg, + action: func(matches []string, value string) { + initOptions(matches[1]) + if b, err := strconv.ParseBool(value); err == nil { + o.Client.Selfhosted[matches[1]].Insecure = b + } + }, }, - selfhostedCAPath: func(matches []string, value string) { - initOptions(matches[1]) - o.Client.Selfhosted[matches[1]].CAPath = value + { + re: selfhostedCAPath, + action: func(matches []string, value string) { + initOptions(matches[1]) + o.Client.Selfhosted[matches[1]].CAPath = value + }, }, } for _, env := range envs { - pair := strings.SplitN(env, "=", 2) - if len(pair) != 2 || len(pair[1]) == 0 { + parts := strings.SplitN(env, "=", 2) + if len(parts) != 2 || parts[1] == "" { continue } + key := strings.ToUpper(parts[0]) + val := parts[1] - key := strings.ToUpper(pair[0]) - value := pair[1] - - for regex, action := range regexActions { - if matches := regex.FindStringSubmatch(key); len(matches) == 2 { - action(matches, value) + for _, p := range regexActions { + if match := p.re.FindStringSubmatch(key); len(match) == 2 { + p.action(match, val) break } } diff --git a/cmd/app/options_test.go b/cmd/app/options_test.go index 0a51aaa2..d8a3c04a 100644 --- a/cmd/app/options_test.go +++ b/cmd/app/options_test.go @@ -2,9 +2,10 @@ package app import ( "os" - "reflect" "testing" + "github.com/stretchr/testify/assert" + "github.com/jetstack/version-checker/pkg/client" "github.com/jetstack/version-checker/pkg/client/acr" "github.com/jetstack/version-checker/pkg/client/docker" @@ -181,10 +182,7 @@ func TestComplete(t *testing.T) { o := new(Options) o.complete() - if !reflect.DeepEqual(o.Client, test.expOptions) { - t.Errorf("unexpected client options, exp=%#+v got=%#+v", - test.expOptions, o.Client) - } + assert.Exactly(t, test.expOptions, o.Client) }) } } @@ -201,7 +199,7 @@ func TestInvalidSelfhostedPanic(t *testing.T) { } for name, test := range tests { t.Run(name, func(t *testing.T) { - defer func() { recover() }() + defer func() { _ = recover() }() o := new(Options) o.assignSelfhosted(test.envs) @@ -236,10 +234,7 @@ func TestInvalidSelfhostedOpts(t *testing.T) { valid := validSelfHostedOpts(&test.opts) - if !reflect.DeepEqual(test.valid, valid) { - t.Errorf("unexpected selfhosted valid options, exp=%#+v got=%#+v", - test.valid, valid) - } + assert.Equal(t, test.valid, valid) }) } } @@ -360,10 +355,7 @@ func TestAssignSelfhosted(t *testing.T) { o := new(Options) o.assignSelfhosted(test.envs) - if !reflect.DeepEqual(o.Client.Selfhosted, test.expOptions.Selfhosted) { - t.Errorf("unexpected client selfhosted options, exp=%#+v got=%#+v", - test.expOptions.Selfhosted, o.Client.Selfhosted) - } + assert.Exactly(t, test.expOptions.Selfhosted, o.Client.Selfhosted) }) } } diff --git a/go.mod b/go.mod index c9253b09..fa6c3aa5 100644 --- a/go.mod +++ b/go.mod @@ -35,19 +35,24 @@ require ( github.com/bombsimon/logrusr/v4 v4.1.0 github.com/go-chi/transport v0.5.0 github.com/gofri/go-github-ratelimit v1.1.1 - github.com/google/go-cmp v0.7.0 github.com/google/go-containerregistry v0.20.3 + github.com/google/go-containerregistry/pkg/authn/k8schain v0.0.0-20250225234217-098045d5e61f github.com/google/go-github/v70 v70.0.0 github.com/hashicorp/go-cleanhttp v0.5.2 github.com/jarcoal/httpmock v1.3.1 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/stretchr/testify v1.10.0 + golang.org/x/time v0.11.0 sigs.k8s.io/controller-runtime v0.20.4 ) require ( + cloud.google.com/go/compute/metadata v0.6.0 // indirect + github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest/azure/auth v0.5.13 // indirect + github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect github.com/Azure/go-autorest/autorest/date v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.2 // indirect github.com/Azure/go-autorest/tracing v0.6.1 // indirect @@ -56,17 +61,21 @@ require ( github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.29.2 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.25.2 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.0 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.33.17 // indirect github.com/aws/smithy-go v1.22.3 // indirect + github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20250115170608-608f37feb051 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 // indirect github.com/containerd/stargz-snapshotter/estargz v0.16.3 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dimchansky/utfbom v1.1.1 // indirect github.com/docker/cli v28.0.4+incompatible // indirect github.com/docker/distribution v2.8.3+incompatible // indirect github.com/docker/docker-credential-helpers v0.9.3 // indirect @@ -84,6 +93,8 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.3 // indirect github.com/google/gnostic-models v0.6.9 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/go-containerregistry/pkg/authn/kubernetes v0.0.0-20241111191718-6bce25ecf029 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect @@ -120,13 +131,11 @@ require ( golang.org/x/sys v0.31.0 // indirect golang.org/x/term v0.30.0 // indirect golang.org/x/text v0.23.0 // indirect - golang.org/x/time v0.11.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect google.golang.org/protobuf v1.36.6 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - gotest.tools/v3 v3.1.0 // indirect k8s.io/apiextensions-apiserver v0.32.3 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect diff --git a/go.sum b/go.sum index 985fee26..55912799 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,22 @@ +cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= +cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= github.com/Azure/go-autorest/autorest v0.11.30 h1:iaZ1RGz/ALZtN5eq4Nr1SOFSlf2E4pDI3Tcsl+dZPVE= github.com/Azure/go-autorest/autorest v0.11.30/go.mod h1:t1kpPIOpIVX7annvothKvb0stsrXa37i7b+xpmBW8Fs= +github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk= github.com/Azure/go-autorest/autorest/adal v0.9.24 h1:BHZfgGsGwdkHDyZdtQRQk1WeUdW0m2WPAwuHZwUi5i4= github.com/Azure/go-autorest/autorest/adal v0.9.24/go.mod h1:7T1+g0PYFmACYW5LlG2fcoPiPlFHjClyRGL7dRlP5c8= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.13 h1:Ov8avRZi2vmrE2JcXw+tu5K/yB41r7xK9GZDiBF7NdM= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.13/go.mod h1:5BAVfWLWXihP47vYrPuBKKf4cS0bXI+KM9Qx6ETDJYo= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 h1:w77/uPk80ZET2F+AfQExZyEWtn+0Rk/uw17m9fv5Ajc= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.6/go.mod h1:piCfgPho7BiIDdEQ1+g4VmKyD5y+p/XtSNqE6Hc4QD0= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/date v0.3.1 h1:o9Z8Jyt+VJJTCZ/UORishuHOusBwolhjokt9s5k8I4w= github.com/Azure/go-autorest/autorest/date v0.3.1/go.mod h1:Dz/RDmXlfiFFS/eW+b/xMUSFs1tboPVy6UjgADToWDM= @@ -39,6 +49,8 @@ github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= github.com/aws/aws-sdk-go-v2/service/ecr v1.43.0 h1:Ak4Ggvvbg8WYxPLoyLOtes1cIMQePvCAi/dUGqm8hOY= github.com/aws/aws-sdk-go-v2/service/ecr v1.43.0/go.mod h1:iQ1skgw1XRK+6Lgkb0I9ODatAP72WoTILh0zXQ5DtbU= +github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.29.2 h1:h4q24ImESGfeamE0I0KJvsblO+03tn8J3+upacKf0vw= +github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.29.2/go.mod h1:3jWiVYuMsv18/qYLY6xVNe84CG/wKaa7vnLaH2/XtxI= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM= @@ -51,6 +63,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.33.17 h1:PZV5W8yk4OtH1JAuhV2PXwwO9v5 github.com/aws/aws-sdk-go-v2/service/sts v1.33.17/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k= github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= +github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20250115170608-608f37feb051 h1:brogdiBXQBvbc+5SQoHOdfxbi77GyaUx6CpuepoEoC4= +github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20250115170608-608f37feb051/go.mod h1:B0Hkcs9+qs/7jvQ+YIIIJ2XKeSbJlkLMEKrz0+Ssgl0= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= @@ -59,6 +73,8 @@ github.com/bombsimon/logrusr/v4 v4.1.0 h1:uZNPbwusB0eUXlO8hIUwStE6Lr5bLN6IgYgG+7 github.com/bombsimon/logrusr/v4 v4.1.0/go.mod h1:pjfHC5e59CvjTBIU3V3sGhFWFAnsnhOR03TRc6im0l8= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 h1:krfRl01rzPzxSxyLyrChD+U+MzsBXbm0OwYYB67uF+4= +github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589/go.mod h1:OuDyvmLnMCwa2ep4Jkm6nyA0ocJuZlGyk2gGseVzERM= github.com/containerd/stargz-snapshotter/estargz v0.16.3 h1:7evrXtoh1mSbGj/pfRccTampEyKpjpOnS3CyiV1Ebr8= github.com/containerd/stargz-snapshotter/estargz v0.16.3/go.mod h1:uyr4BfYfOj3G9WBVE8cOlQmXAbPN9VEQpBBeJIuOipU= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= @@ -68,6 +84,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= +github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/docker/cli v28.0.4+incompatible h1:pBJSJeNd9QeIWPjRcV91RVJihd/TXB77q1ef64XEu4A= github.com/docker/cli v28.0.4+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= @@ -76,8 +94,8 @@ github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqI github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo= github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= -github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= @@ -107,6 +125,7 @@ github.com/gofri/go-github-ratelimit v1.1.1/go.mod h1:wGZlBbzHmIVjwDR3pZgKY7RBTV github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= @@ -119,12 +138,15 @@ github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl76 github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-containerregistry v0.20.3 h1:oNx7IdTI936V8CQRveCjaxOiegWwvM7kqkbXTpyiovI= github.com/google/go-containerregistry v0.20.3/go.mod h1:w00pIgBRDVUDFM6bq+Qx8lwNWK+cxgCuX1vd3PIBDNI= +github.com/google/go-containerregistry/pkg/authn/k8schain v0.0.0-20250225234217-098045d5e61f h1:LA+8uYrQl2biusGs1VEnIUQHLu8RjaCUNqHsieRkaTI= +github.com/google/go-containerregistry/pkg/authn/k8schain v0.0.0-20250225234217-098045d5e61f/go.mod h1:8mk2eu7HGqCp+JSWQVFCnKQwk/K6cIY6ID9aX72iTRo= +github.com/google/go-containerregistry/pkg/authn/kubernetes v0.0.0-20241111191718-6bce25ecf029 h1:tmtax9EjrCFrrw72NeGso7qZUnJXTIP368kcjE4lZwE= +github.com/google/go-containerregistry/pkg/authn/kubernetes v0.0.0-20241111191718-6bce25ecf029/go.mod h1:zD6WJVa49IK5fhrZOUaq7UcSgxZFlnS80EJBrcVFkFI= github.com/google/go-github/v70 v70.0.0 h1:/tqCp5KPrcvqCc7vIvYyFYTiCGrYvaWoYMGHSQbo55o= github.com/google/go-github/v70 v70.0.0/go.mod h1:xBUZgo8MI3lUL/hwxl3hlceJW1U8MVnXP3zUyI+rhQY= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= @@ -221,7 +243,6 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -290,7 +311,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -324,7 +344,6 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= diff --git a/pkg/api/options.go b/pkg/api/options.go index 8590aff1..6b66d162 100644 --- a/pkg/api/options.go +++ b/pkg/api/options.go @@ -7,20 +7,20 @@ import "regexp" type Options struct { OverrideURL *string `json:"override-url,omitempty"` + MatchRegex *string `json:"match-regex,omitempty"` + + PinMajor *int64 `json:"pin-major,omitempty"` + PinMinor *int64 `json:"pin-minor,omitempty"` + PinPatch *int64 `json:"pin-patch,omitempty"` + + RegexMatcher *regexp.Regexp `json:"-"` + // UseSHA cannot be used with any other options UseSHA bool `json:"use-sha,omitempty"` // Resolve SHA to a TAG ResolveSHAToTags bool `json:"resolve-sha-to-tags,omitempty"` - MatchRegex *string `json:"match-regex,omitempty"` - // UseMetaData defines whether tags with '-alpha', '-debian.0' etc. is // permissible. UseMetaData bool `json:"use-metadata,omitempty"` - - PinMajor *int64 `json:"pin-major,omitempty"` - PinMinor *int64 `json:"pin-minor,omitempty"` - PinPatch *int64 `json:"pin-patch,omitempty"` - - RegexMatcher *regexp.Regexp `json:"-"` } diff --git a/pkg/api/types.go b/pkg/api/types.go index 903745a1..196e79ef 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -12,6 +12,30 @@ type ImageTag struct { Timestamp time.Time `json:"timestamp"` OS OS `json:"os,omitempty"` Architecture Architecture `json:"architecture,omitempty"` + + // If this is a Manifest list we need to keep them together + Children []*ImageTag `json:"children,omitempty"` +} + +func (i *ImageTag) HasChildren() bool { + return len(i.Children) > 0 +} + +func (i *ImageTag) MatchesSHA(sha string) bool { + if sha == i.SHA { + return true + } + for _, known := range i.Children { + if known.SHA == sha { + return true + } + } + return false +} + +type Platform struct { + OS OS + Architecture Architecture } type OS string diff --git a/pkg/client/acr/acr.go b/pkg/client/acr/acr.go index fa21e4da..cf91a88b 100644 --- a/pkg/client/acr/acr.go +++ b/pkg/client/acr/acr.go @@ -8,33 +8,26 @@ import ( "io" "net/http" "sync" - "time" - - "github.com/MicahParks/keyfunc/v3" "github.com/Azure/go-autorest/autorest" - "github.com/Azure/go-autorest/autorest/adal" - "github.com/golang-jwt/jwt/v5" "github.com/jetstack/version-checker/pkg/api" "github.com/jetstack/version-checker/pkg/client/util" ) +// Ensure that we are an ImageClient +var _ api.ImageClient = (*Client)(nil) + const ( userAgent = "jetstack/version-checker" requiredScope = "repository:*:metadata_read" ) type Client struct { - Options - - cacheMu sync.Mutex cachedACRClient map[string]*acrClient -} + Options -type acrClient struct { - tokenExpiry time.Time - *autorest.Client + cacheMu sync.Mutex } type Options struct { @@ -44,18 +37,6 @@ type Options struct { JWKSURI string } -type AccessTokenResponse struct { - AccessToken string `json:"access_token"` -} - -type ManifestResponse struct { - Manifests []struct { - Digest string `json:"digest"` - CreatedTime time.Time `json:"createdTime"` - Tags []string `json:"tags"` - } `json:"manifests"` -} - func New(opts Options) (*Client, error) { if len(opts.RefreshToken) > 0 && (len(opts.Username) > 0 || len(opts.Password) > 0) { @@ -90,27 +71,33 @@ func (c *Client) Tags(ctx context.Context, host, repo, image string) ([]api.Imag host, err) } - var tags []api.ImageTag + // Create a map of tags, so that when we come up with additional Tags + // we can add them as Children + tags := map[string]api.ImageTag{} + for _, manifest := range manifestResp.Manifests { - if len(manifest.Tags) == 0 { - tags = append(tags, api.ImageTag{ - SHA: manifest.Digest, - Timestamp: manifest.CreatedTime, - }) + // Base data shared across tags + base := api.ImageTag{ + SHA: manifest.Digest, + Timestamp: manifest.CreatedTime, + OS: manifest.OS, + Architecture: manifest.Architecture, + } + // No tags, use digest as the key + if len(manifest.Tags) == 0 { + tags[base.SHA] = base continue } for _, tag := range manifest.Tags { - tags = append(tags, api.ImageTag{ - SHA: manifest.Digest, - Timestamp: manifest.CreatedTime, - Tag: tag, - }) + current := base // copy the base + current.Tag = tag // set tag value + + util.FindParentTags(tags, tag, ¤t) } } - - return tags, nil + return util.TagMaptoList(tags), nil } func (c *Client) getManifestsWithClient(ctx context.Context, client *acrClient, host, repo, image string) (*http.Response, error) { @@ -148,147 +135,3 @@ func (c *Client) getManifestsWithClient(ctx context.Context, client *acrClient, return resp, nil } - -func (c *Client) getACRClient(ctx context.Context, host string) (*acrClient, error) { - c.cacheMu.Lock() - defer c.cacheMu.Unlock() - - if client, ok := c.cachedACRClient[host]; ok && time.Now().After(client.tokenExpiry) { - return client, nil - } - - var ( - client *acrClient - accessTokenClient *autorest.Client - accessTokenReq *http.Request - err error - ) - if len(c.RefreshToken) > 0 { - accessTokenClient, accessTokenReq, err = c.getAccessTokenRequesterForRefreshToken(ctx, host) - } else { - accessTokenClient, accessTokenReq, err = c.getAccessTokenRequesterForBasicAuth(ctx, host) - } - if err != nil { - return nil, err - } - if client, err = c.getAuthorizedClient(accessTokenClient, accessTokenReq, host); err != nil { - return nil, err - } - - c.cachedACRClient[host] = client - - return client, nil -} - -func (c *Client) getAccessTokenRequesterForBasicAuth(ctx context.Context, host string) (*autorest.Client, *http.Request, error) { - client := autorest.NewClientWithUserAgent(userAgent) - client.Authorizer = autorest.NewBasicAuthorizer(c.Username, c.Password) - urlParameters := map[string]interface{}{ - "url": "https://" + host, - } - - preparer := autorest.CreatePreparer( - autorest.WithCustomBaseURL("{url}", urlParameters), - autorest.WithPath("/oauth2/token"), - autorest.WithQueryParameters(map[string]interface{}{ - "scope": requiredScope, - "service": host, - }), - ) - req, err := preparer.Prepare((&http.Request{}).WithContext(ctx)) - if err != nil { - return nil, nil, err - } - - return &client, req, nil -} - -func (c *Client) getAccessTokenRequesterForRefreshToken(ctx context.Context, host string) (*autorest.Client, *http.Request, error) { - client := autorest.NewClientWithUserAgent(userAgent) - urlParameters := map[string]interface{}{ - "url": "https://" + host, - } - - formDataParameters := map[string]interface{}{ - "grant_type": "refresh_token", - "refresh_token": c.RefreshToken, - "scope": requiredScope, - "service": host, - } - - preparer := autorest.CreatePreparer( - autorest.AsPost(), - autorest.WithCustomBaseURL("{url}", urlParameters), - autorest.WithPath("/oauth2/token"), - autorest.WithFormData(autorest.MapToValues(formDataParameters))) - req, err := preparer.Prepare((&http.Request{}).WithContext(ctx)) - if err != nil { - return nil, nil, err - } - return &client, req, nil -} - -func (c *Client) getAuthorizedClient(client *autorest.Client, req *http.Request, host string) (*acrClient, error) { - resp, err := autorest.SendWithSender(client, req, - autorest.DoRetryForStatusCodes(client.RetryAttempts, client.RetryDuration, autorest.StatusCodesForRetry...), - ) - if err != nil { - return nil, fmt.Errorf("%s: failed to request access token: %s", - host, err) - } - defer func() { _ = resp.Body.Close() }() - - var respToken AccessTokenResponse - if err := json.NewDecoder(resp.Body).Decode(&respToken); err != nil { - return nil, fmt.Errorf("%s: failed to decode access token response: %s", - host, err) - } - - exp, err := c.getTokenExpiration(respToken.AccessToken) - if err != nil { - return nil, fmt.Errorf("%s: %s", host, err) - } - - token := &adal.Token{ - RefreshToken: "", // empty if access_token was retrieved with basic auth. but client is not reused after expiry anyway (see cachedACRClient) - AccessToken: respToken.AccessToken, - } - - client.Authorizer = autorest.NewBearerAuthorizer(token) - - return &acrClient{ - tokenExpiry: exp, - Client: client, - }, nil -} - -func (c *Client) getTokenExpiration(tokenString string) (time.Time, error) { - jwtParser := jwt.NewParser(jwt.WithoutClaimsValidation()) - var token *jwt.Token - var err error - if c.JWKSURI != "" { - var k keyfunc.Keyfunc - k, err = keyfunc.NewDefaultCtx(context.TODO(), []string{c.JWKSURI}) - if err != nil { - return time.Time{}, err - } - token, err = jwtParser.Parse(tokenString, k.Keyfunc) - } else { - token, _, err = jwtParser.ParseUnverified(tokenString, jwt.MapClaims{}) - } - if err != nil { - return time.Time{}, err - } - - claims, ok := token.Claims.(jwt.MapClaims) - if !ok { - return time.Time{}, fmt.Errorf("failed to process claims in access token") - } - - if exp, ok := claims["exp"].(float64); ok { - timestamp := time.Unix(int64(exp), 0) - return timestamp, nil - } - - return time.Time{}, fmt.Errorf("failed to find 'exp' claim in access token") -} diff --git a/pkg/client/acr/auth.go b/pkg/client/acr/auth.go new file mode 100644 index 00000000..5461c3fd --- /dev/null +++ b/pkg/client/acr/auth.go @@ -0,0 +1,163 @@ +package acr + +// The intention here is to provide a client for Azure Container Registry (ACR) +// that can authenticate using either basic authentication (username/password) +// or a refresh token. The client will cache the access token and its expiration +// time to avoid unnecessary requests to the ACR server. + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/Azure/go-autorest/autorest" + "github.com/Azure/go-autorest/autorest/adal" + "github.com/MicahParks/keyfunc/v3" + "github.com/golang-jwt/jwt/v5" +) + +func (c *Client) getACRClient(ctx context.Context, host string) (*acrClient, error) { + c.cacheMu.Lock() + defer c.cacheMu.Unlock() + + if client, ok := c.cachedACRClient[host]; ok && time.Now().After(client.tokenExpiry) { + return client, nil + } + + var ( + client *acrClient + accessTokenClient *autorest.Client + accessTokenReq *http.Request + err error + ) + if len(c.RefreshToken) > 0 { + accessTokenClient, accessTokenReq, err = c.getAccessTokenRequesterForRefreshToken(ctx, host) + } else { + accessTokenClient, accessTokenReq, err = c.getAccessTokenRequesterForBasicAuth(ctx, host) + } + if err != nil { + return nil, err + } + if client, err = c.getAuthorizedClient(accessTokenClient, accessTokenReq, host); err != nil { + return nil, err + } + + c.cachedACRClient[host] = client + + return client, nil +} + +func (c *Client) getAccessTokenRequesterForBasicAuth(ctx context.Context, host string) (*autorest.Client, *http.Request, error) { + client := autorest.NewClientWithUserAgent(userAgent) + client.Authorizer = autorest.NewBasicAuthorizer(c.Username, c.Password) + urlParameters := map[string]interface{}{ + "url": "https://" + host, + } + + preparer := autorest.CreatePreparer( + autorest.WithCustomBaseURL("{url}", urlParameters), + autorest.WithPath("/oauth2/token"), + autorest.WithQueryParameters(map[string]interface{}{ + "scope": requiredScope, + "service": host, + }), + ) + req, err := preparer.Prepare((&http.Request{}).WithContext(ctx)) + if err != nil { + return nil, nil, err + } + + return &client, req, nil +} + +func (c *Client) getAccessTokenRequesterForRefreshToken(ctx context.Context, host string) (*autorest.Client, *http.Request, error) { + client := autorest.NewClientWithUserAgent(userAgent) + urlParameters := map[string]interface{}{ + "url": "https://" + host, + } + + formDataParameters := map[string]interface{}{ + "grant_type": "refresh_token", + "refresh_token": c.RefreshToken, + "scope": requiredScope, + "service": host, + } + + preparer := autorest.CreatePreparer( + autorest.AsPost(), + autorest.WithCustomBaseURL("{url}", urlParameters), + autorest.WithPath("/oauth2/token"), + autorest.WithFormData(autorest.MapToValues(formDataParameters))) + req, err := preparer.Prepare((&http.Request{}).WithContext(ctx)) + if err != nil { + return nil, nil, err + } + return &client, req, nil +} + +func (c *Client) getAuthorizedClient(client *autorest.Client, req *http.Request, host string) (*acrClient, error) { + resp, err := autorest.SendWithSender(client, req, + autorest.DoRetryForStatusCodes(client.RetryAttempts, client.RetryDuration, autorest.StatusCodesForRetry...), + ) + if err != nil { + return nil, fmt.Errorf("%s: failed to request access token: %s", + host, err) + } + defer func() { _ = resp.Body.Close() }() + + var respToken AccessTokenResponse + if err := json.NewDecoder(resp.Body).Decode(&respToken); err != nil { + return nil, fmt.Errorf("%s: failed to decode access token response: %s", + host, err) + } + + exp, err := c.getTokenExpiration(respToken.AccessToken) + if err != nil { + return nil, fmt.Errorf("%s: %s", host, err) + } + + token := &adal.Token{ + RefreshToken: "", // empty if access_token was retrieved with basic auth. but client is not reused after expiry anyway (see cachedACRClient) + AccessToken: respToken.AccessToken, + } + + client.Authorizer = autorest.NewBearerAuthorizer(token) + + return &acrClient{ + tokenExpiry: exp, + Client: client, + }, nil +} + +func (c *Client) getTokenExpiration(tokenString string) (time.Time, error) { + jwtParser := jwt.NewParser(jwt.WithoutClaimsValidation()) + var token *jwt.Token + var err error + if c.JWKSURI != "" { + var k keyfunc.Keyfunc + k, err = keyfunc.NewDefaultCtx(context.TODO(), []string{c.JWKSURI}) + if err != nil { + return time.Time{}, err + } + token, err = jwtParser.Parse(tokenString, k.Keyfunc) + } else { + token, _, err = jwtParser.ParseUnverified(tokenString, jwt.MapClaims{}) + } + if err != nil { + return time.Time{}, err + } + + claims, ok := token.Claims.(jwt.MapClaims) + if !ok { + return time.Time{}, fmt.Errorf("failed to process claims in access token") + } + + if exp, ok := claims["exp"].(float64); ok { + timestamp := time.Unix(int64(exp), 0) + return timestamp, nil + } + + return time.Time{}, fmt.Errorf("failed to find 'exp' claim in access token") +} diff --git a/pkg/client/acr/types.go b/pkg/client/acr/types.go new file mode 100644 index 00000000..bd09cbb5 --- /dev/null +++ b/pkg/client/acr/types.go @@ -0,0 +1,30 @@ +package acr + +import ( + "time" + + "github.com/Azure/go-autorest/autorest" + "github.com/jetstack/version-checker/pkg/api" +) + +type acrClient struct { + tokenExpiry time.Time + *autorest.Client +} + +type AccessTokenResponse struct { + AccessToken string `json:"access_token"` +} + +// API Taken from documentation @ +// https://learn.microsoft.com/en-us/rest/api/containerregistry/manifests/get-list?view=rest-containerregistry-2019-08-15&tabs=HTTP + +type ManifestResponse struct { + Manifests []struct { + CreatedTime time.Time `json:"createdTime"` + Digest string `json:"digest"` + Architecture api.Architecture `json:"architecture,omitempty"` + OS api.OS `json:"os,omitempty"` + Tags []string `json:"tags"` + } `json:"manifests"` +} diff --git a/pkg/client/client.go b/pkg/client/client.go index fd54cd08..219802d6 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -28,21 +28,21 @@ type ClientHandler interface { // Client is a container image registry client to list tags of given image // URLs. type Client struct { - clients []api.ImageClient fallbackClient api.ImageClient - log *logrus.Entry + log *logrus.Entry + clients []api.ImageClient } // Options used to configure client authentication. type Options struct { ACR acr.Options + Docker docker.Options ECR ecr.Options GCR gcr.Options GHCR ghcr.Options - Docker docker.Options - Quay quay.Options OCI oci.Options + Quay quay.Options Selfhosted map[string]*selfhosted.Options Transport http.RoundTripper @@ -79,7 +79,7 @@ func New(ctx context.Context, log *logrus.Entry, opts Options) (*Client, error) } // Create some of the fallback clients - ociclient, err := oci.New(&opts.OCI) + ociclient, err := oci.New(&opts.OCI, log) if err != nil { return nil, fmt.Errorf("failed to create OCI client: %w", err) } diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 7e32945f..61a6081b 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -2,10 +2,10 @@ package client import ( "context" - "reflect" "testing" "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" "github.com/jetstack/version-checker/pkg/api" "github.com/jetstack/version-checker/pkg/client/acr" @@ -181,20 +181,10 @@ func TestFromImageURL(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { client, host, path := handler.fromImageURL(test.url) - if reflect.TypeOf(client) != reflect.TypeOf(test.expClient) { - t.Errorf("unexpected client, exp=%v got=%v", - reflect.TypeOf(test.expClient), reflect.TypeOf(client)) - } - if host != test.expHost { - t.Errorf("unexpected host, exp=%v got=%v", - test.expHost, host) - } - - if path != test.expPath { - t.Errorf("unexpected path, exp=%s got=%s", - test.expPath, path) - } + assert.IsType(t, test.expClient, client) + assert.Equal(t, test.expHost, host) + assert.Equal(t, test.expPath, path) }) } } diff --git a/pkg/client/docker/docker.go b/pkg/client/docker/docker.go index 4662f3af..b6322336 100644 --- a/pkg/client/docker/docker.go +++ b/pkg/client/docker/docker.go @@ -10,10 +10,23 @@ import ( "strings" "time" + "golang.org/x/time/rate" + "github.com/sirupsen/logrus" "github.com/hashicorp/go-retryablehttp" "github.com/jetstack/version-checker/pkg/api" + "github.com/jetstack/version-checker/pkg/client/util" +) + +// Ensure that we are an ImageClient +var _ api.ImageClient = (*Client)(nil) + +// Values taken from: https://docs.docker.com/docker-hub/usage/#abuse-rate-limit +const ( + windowDuration = time.Minute + APIRateLimit = 500 + maxWait = time.Hour ) const ( @@ -22,28 +35,39 @@ const ( ) type Options struct { + Transporter http.RoundTripper Username string Password string Token string - Transporter http.RoundTripper } type Client struct { *http.Client Options + + log *logrus.Entry + limiter *rate.Limiter } func New(opts Options, log *logrus.Entry) (*Client, error) { ctx := context.Background() + + limiter := rate.NewLimiter( + rate.Every(windowDuration/APIRateLimit), + 1, + ) + log = log.WithField("client", "docker") + retryclient := retryablehttp.NewClient() if opts.Transporter != nil { retryclient.HTTPClient.Transport = opts.Transporter } + retryclient.Backoff = util.RateLimitedBackoffLimiter(log, limiter, maxWait) retryclient.HTTPClient.Timeout = 10 * time.Second retryclient.RetryMax = 10 retryclient.RetryWaitMax = 2 * time.Minute retryclient.RetryWaitMin = 1 * time.Second - retryclient.Logger = log.WithField("client", "docker") + retryclient.Logger = log client := retryclient.StandardClient() // Setup Auth if username and password used. @@ -62,6 +86,8 @@ func New(opts Options, log *logrus.Entry) (*Client, error) { return &Client{ Options: opts, Client: client, + log: log, + limiter: limiter, }, nil } @@ -93,13 +119,23 @@ func (c *Client) Tags(ctx context.Context, _, repo, image string) ([]api.ImageTa } } + tag := api.ImageTag{ + Tag: result.Name, + Timestamp: timestamp, + } + + // If we have a Digest, lets set it.. + if result.Digest != "" { + tag.SHA = result.Digest + } + for _, image := range result.Images { // Image without digest contains no real image. if len(image.Digest) == 0 { continue } - tags = append(tags, api.ImageTag{ + tag.Children = append(tag.Children, &api.ImageTag{ Tag: result.Name, SHA: image.Digest, Timestamp: timestamp, @@ -107,6 +143,14 @@ func (c *Client) Tags(ctx context.Context, _, repo, image string) ([]api.ImageTa Architecture: image.Architecture, }) } + + // If we only have one child, and it has a SHA, then lets use that in the parent + if tag.SHA == "" && len(tag.Children) == 1 && tag.Children[0].SHA != "" { + tag.SHA = tag.Children[0].SHA + } + + // Append our Tag at the end... + tags = append(tags, tag) } url = response.Next @@ -126,6 +170,7 @@ func (c *Client) doRequest(ctx context.Context, url string) (*TagResponse, error if len(c.Token) > 0 { req.Header.Add("Authorization", "Bearer "+c.Token) } + req.Header.Set("User-Agent", "version-checker/docker") resp, err := c.Do(req) if err != nil { @@ -158,6 +203,7 @@ func basicAuthSetup(ctx context.Context, client *http.Client, opts Options) (str return "", err } + req.Header.Set("User-Agent", "version-checker/docker") req.Header.Set("Content-Type", "application/json") req = req.WithContext(ctx) diff --git a/pkg/client/docker/docker_test.go b/pkg/client/docker/docker_test.go new file mode 100644 index 00000000..5b3b7fcd --- /dev/null +++ b/pkg/client/docker/docker_test.go @@ -0,0 +1,138 @@ +package docker + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/jetstack/version-checker/pkg/api" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/sirupsen/logrus" +) + +type hostnameOverride struct { + Host string + RT http.RoundTripper +} + +func (r *hostnameOverride) RoundTrip(req *http.Request) (*http.Response, error) { + if req.Host != r.Host { + if testing.Verbose() { + fmt.Printf("Overriding URI from: %s to %s\n", req.Host, strings.TrimPrefix(r.Host, "http://")) + } + req.Host = strings.TrimPrefix(r.Host, "http://") + req.URL.Host = strings.TrimPrefix(r.Host, "http://") + req.URL.Scheme = "http" + } + // fmt.Printf("Req: %+v", req) + return r.RT.RoundTrip(req) +} + +func TestTags(t *testing.T) { + log := logrus.NewEntry(logrus.New()) + ctx := context.Background() + + t.Run("successful Tags fetch", func(t *testing.T) { + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.NotEmpty(t, r.Header.Get("Authorization")) + + require.Equal(t, "/v2/repositories/testrepo/testimage/tags", r.URL.Path) + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(TagResponse{ + Results: []Result{ + { + Name: "v1.0.0", + Timestamp: time.Now().Add(-24 * time.Hour).Format(time.RFC3339Nano), + Digest: "sha256:abcdef", + Images: []Image{ + {Digest: "sha256:child1", OS: "linux", Architecture: "amd64"}, + }, + }, + { + Name: "v2.0.0", + Timestamp: time.Now().Add(-48 * time.Hour).Format(time.RFC3339Nano), + Images: []Image{ + {Digest: "sha256:child2", OS: "linux", Architecture: "amd64"}, + }, + }, + }, + }) + })) + defer server.Close() + + client := &Client{ + Client: server.Client(), + log: log, + Options: Options{ + Token: "testtoken", + }, + } + + client.Transport = &hostnameOverride{RT: server.Client().Transport, Host: server.URL} + + tags, err := client.Tags(ctx, "NOT USED!", "testrepo", "testimage") + require.NoError(t, err) + require.Len(t, tags, 2) + + assert.Equal(t, "v1.0.0", tags[0].Tag) + assert.Equal(t, "sha256:abcdef", tags[0].SHA) + assert.Equal(t, api.OS("linux"), tags[0].Children[0].OS) + assert.Equal(t, api.Architecture("amd64"), tags[0].Children[0].Architecture) + + assert.Equal(t, "v2.0.0", tags[1].Tag) + assert.NotEmpty(t, tags[1].SHA) + }) + + t.Run("error on invalid response", func(t *testing.T) { + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("invalid json")) + })) + defer server.Close() + + client := &Client{ + Client: server.Client(), + log: log, + Options: Options{ + Token: "testtoken", + }, + } + client.Transport = &hostnameOverride{RT: server.Client().Transport, Host: server.URL} + + tags, err := client.Tags(ctx, "NOT USED!", "testrepo", "testimage") + assert.Nil(t, tags) + assert.Error(t, err) + assert.Contains(t, err.Error(), "unexpected image tags response") + }) + + t.Run("error on non-200 status code", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte("not found")) + })) + defer server.Close() + + client := &Client{ + Client: server.Client(), + log: log, + Options: Options{ + Token: "testtoken", + }, + } + client.Transport = &hostnameOverride{RT: server.Client().Transport, Host: server.URL} + + tags, err := client.Tags(ctx, "NOT USED!", "testrepo", "testimage") + assert.Nil(t, tags) + assert.Error(t, err) + assert.Contains(t, err.Error(), "unexpected image") + }) +} diff --git a/pkg/client/docker/types.go b/pkg/client/docker/types.go index bfeaa994..cd5ae877 100644 --- a/pkg/client/docker/types.go +++ b/pkg/client/docker/types.go @@ -12,9 +12,14 @@ type TagResponse struct { } type Result struct { - Name string `json:"name"` - Timestamp string `json:"last_updated"` - Images []Image `json:"images"` + Name string `json:"name"` + Timestamp string `json:"last_updated"` + TagStatus string `json:"tag_status"` // String of "active" or "inactive" + MediaType string `json:"media_type,omitempty"` + // Digest is only set with `application/vnd.oci.image.index.v1+json` media_type + Digest string `json:"digest,omitempty"` + + Images []Image `json:"images"` } type Image struct { diff --git a/pkg/client/ecr/ecr.go b/pkg/client/ecr/ecr.go index 3654fee4..431065d1 100644 --- a/pkg/client/ecr/ecr.go +++ b/pkg/client/ecr/ecr.go @@ -14,18 +14,20 @@ import ( "github.com/jetstack/version-checker/pkg/client/util" ) -type Client struct { - Config aws.Config +// Ensure that we are an ImageClient +var _ api.ImageClient = (*Client)(nil) +type Client struct { Options + Config aws.Config } type Options struct { + Transporter http.RoundTripper IamRoleArn string AccessKeyID string SecretAccessKey string SessionToken string - Transporter http.RoundTripper } func New(opts Options) *Client { @@ -64,28 +66,29 @@ func (c *Client) Tags(ctx context.Context, host, repo, image string) ([]api.Imag return nil, fmt.Errorf("failed to describe images: %s", err) } - var tags []api.ImageTag + tags := map[string]api.ImageTag{} for _, img := range images.ImageDetails { + // Base data shared across tags + base := api.ImageTag{ + SHA: *img.ImageDigest, + Timestamp: *img.ImagePushedAt, + } + // Continue early if no tags available if len(img.ImageTags) == 0 { - tags = append(tags, api.ImageTag{ - SHA: *img.ImageDigest, - Timestamp: *img.ImagePushedAt, - }) - + tags[base.SHA] = base continue } for _, tag := range img.ImageTags { - tags = append(tags, api.ImageTag{ - SHA: *img.ImageDigest, - Timestamp: *img.ImagePushedAt, - Tag: tag, - }) + current := base // copy the base + current.Tag = tag // set tag value + + util.FindParentTags(tags, tag, ¤t) } } - return tags, nil + return util.TagMaptoList(tags), nil } func (c *Client) createClient(ctx context.Context, region string) (*ecr.Client, error) { diff --git a/pkg/client/fallback/fallback.go b/pkg/client/fallback/fallback.go index 12fc0c52..19748215 100644 --- a/pkg/client/fallback/fallback.go +++ b/pkg/client/fallback/fallback.go @@ -12,11 +12,13 @@ import ( "github.com/sirupsen/logrus" ) -type Client struct { - clients []api.ImageClient +// Ensure that we are an ImageClient +var _ api.ImageClient = (*Client)(nil) +type Client struct { log *logrus.Entry hostCache *cache.Cache + clients []api.ImageClient } func New(ctx context.Context, log *logrus.Entry, clients []api.ImageClient) (*Client, error) { diff --git a/pkg/client/gcr/gcr.go b/pkg/client/gcr/gcr.go index 15ebccc6..67eefaab 100644 --- a/pkg/client/gcr/gcr.go +++ b/pkg/client/gcr/gcr.go @@ -16,9 +16,12 @@ const ( lookupURL = "https://%s/v2/%s/tags/list" ) +// Ensure that we are an ImageClient +var _ api.ImageClient = (*Client)(nil) + type Options struct { - Token string Transporter http.RoundTripper + Token string } type Client struct { @@ -28,11 +31,12 @@ type Client struct { type Response struct { Manifest map[string]ManifestItem `json:"manifest"` + Tags []string `json:"tags,omitempty"` } type ManifestItem struct { - Tag []string `json:"tag"` TimeCreated string `json:"timeCreatedMs"` + Tags []string `json:"tag"` } func New(opts Options) *Client { @@ -98,24 +102,47 @@ func (c *Client) buildRequest(ctx context.Context, url string) (*http.Request, e } func (c *Client) extractImageTags(response Response) ([]api.ImageTag, error) { - var tags []api.ImageTag + tags := map[string]api.ImageTag{} for sha, manifestItem := range response.Manifest { timestamp, err := c.convertTimestamp(manifestItem.TimeCreated) if err != nil { return nil, fmt.Errorf("failed to convert timestamp string: %w", err) } + // Base data shared across tags + base := api.ImageTag{ + SHA: sha, + Timestamp: timestamp, + } + // If no tag, add without and continue early. - if len(manifestItem.Tag) == 0 { - tags = append(tags, api.ImageTag{SHA: sha, Timestamp: timestamp}) + if len(manifestItem.Tags) == 0 { + tags[sha] = base continue } - for _, tag := range manifestItem.Tag { - tags = append(tags, api.ImageTag{Tag: tag, SHA: sha, Timestamp: timestamp}) + for _, tag := range manifestItem.Tags { + current := base // copy the base + current.Tag = tag // set tag value + + // Already exists — add as child + if parent, exists := tags[tag]; exists { + parent.Children = append(parent.Children, ¤t) + tags[tag] = parent + } else { + // First occurrence — assign as root + tags[tag] = current + } } } - return tags, nil + + // Convert Map to Slice + taglist := make([]api.ImageTag, 0, len(tags)) + for _, tag := range tags { + taglist = append(taglist, tag) + } + + return taglist, nil } func (c *Client) convertTimestamp(timeCreated string) (time.Time, error) { diff --git a/pkg/client/gcr/path.go b/pkg/client/gcr/path.go index 69b678ea..7673e6ac 100644 --- a/pkg/client/gcr/path.go +++ b/pkg/client/gcr/path.go @@ -14,12 +14,16 @@ func (c *Client) IsHost(host string) bool { } func (c *Client) RepoImageFromPath(path string) (string, string) { - lastIndex := strings.LastIndex(path, "/") + split := strings.Split(path, "/") - // If there's no slash, then its a "root" level image - if lastIndex == -1 { - return "", path + lenSplit := len(split) + if lenSplit == 1 { + return "google-containers", split[0] } - return path[:lastIndex], path[lastIndex+1:] + if lenSplit > 1 { + return strings.Join(split[:len(split)-1], "/"), split[lenSplit-1] + } + + return path, "" } diff --git a/pkg/client/gcr/path_test.go b/pkg/client/gcr/path_test.go index 703c59a7..7bc80e47 100644 --- a/pkg/client/gcr/path_test.go +++ b/pkg/client/gcr/path_test.go @@ -1,6 +1,10 @@ package gcr -import "testing" +import ( + "testing" + + "github.com/stretchr/testify/assert" +) func TestIsHost(t *testing.T) { tests := map[string]struct { @@ -60,10 +64,9 @@ func TestIsHost(t *testing.T) { handler := new(Client) for name, test := range tests { t.Run(name, func(t *testing.T) { - if isHost := handler.IsHost(test.host); isHost != test.expIs { - t.Errorf("%s: unexpected IsHost, exp=%t got=%t", - test.host, test.expIs, isHost) - } + assert.Equal(t, test.expIs, + handler.IsHost(test.host), + ) }) } } @@ -94,10 +97,8 @@ func TestRepoImage(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { repo, image := handler.RepoImageFromPath(test.path) - if repo != test.expRepo && image != test.expImage { - t.Errorf("%s: unexpected repo/image, exp=%s/%s got=%s/%s", - test.path, test.expRepo, test.expImage, repo, image) - } + assert.Equal(t, test.expRepo, repo) + assert.Equal(t, test.expImage, image) }) } } diff --git a/pkg/client/ghcr/ghcr.go b/pkg/client/ghcr/ghcr.go index 40421d7e..80a2e5e2 100644 --- a/pkg/client/ghcr/ghcr.go +++ b/pkg/client/ghcr/ghcr.go @@ -13,16 +13,19 @@ import ( "github.com/google/go-github/v70/github" ) +// Ensure that we are an ImageClient +var _ api.ImageClient = (*Client)(nil) + type Options struct { + Transporter http.RoundTripper Token string Hostname string - Transporter http.RoundTripper } type Client struct { client *github.Client - opts Options ownerTypes map[string]string + opts Options } func New(opts Options) *Client { @@ -106,28 +109,43 @@ func (c *Client) buildPackageListOptions() *github.PackageListOptions { } func (c *Client) extractImageTags(versions []*github.PackageVersion) []api.ImageTag { - var tags []api.ImageTag + tags := map[string]api.ImageTag{} for _, ver := range versions { if meta, ok := ver.GetMetadata(); ok { - if len(meta.Container.Tags) == 0 { - continue - } - sha := "" + var sha string if strings.HasPrefix(*ver.Name, "sha") { sha = *ver.Name } + base := api.ImageTag{ + Tag: *ver.Name, + SHA: sha, + Timestamp: ver.CreatedAt.Time, + } + for _, tag := range meta.Container.Tags { - tags = append(tags, api.ImageTag{ - Tag: tag, - SHA: sha, - Timestamp: ver.CreatedAt.Time, - }) + current := base // copy the base + current.Tag = tag // set tag value + + // Tag Already exists — add as child + if parent, exists := tags[tag]; exists { + parent.Children = append(parent.Children, ¤t) + tags[tag] = parent + } else { + // First occurrence of Tag — assign as root + tags[tag] = current + } } } } - return tags + + // Convert Map to Slice + taglist := make([]api.ImageTag, 0, len(tags)) + for _, tag := range tags { + taglist = append(taglist, tag) + } + return taglist } func (c *Client) ownerType(ctx context.Context, owner string) (string, error) { diff --git a/pkg/client/ghcr/ghcr_test.go b/pkg/client/ghcr/ghcr_test.go index 39c2f6b6..ff9568f3 100644 --- a/pkg/client/ghcr/ghcr_test.go +++ b/pkg/client/ghcr/ghcr_test.go @@ -78,8 +78,7 @@ func TestClient_Tags(t *testing.T) { tags, err := client.Tags(ctx, host, "test-user-owner", "test-repo") assert.NoError(t, err) assert.Len(t, tags, 2) - assert.Equal(t, "tag1", tags[0].Tag) - assert.Equal(t, "tag2", tags[1].Tag) + assert.ElementsMatch(t, []string{"tag1", "tag2"}, []string{tags[0].Tag, tags[1].Tag}) }) t.Run("failed to fetch owner type", func(t *testing.T) { @@ -146,8 +145,7 @@ func TestClient_Tags(t *testing.T) { tags, err := client.Tags(ctx, host, "test-user-owner", "test-repo") assert.NoError(t, err) assert.Len(t, tags, 2) - assert.Equal(t, "tag1", tags[0].Tag) - assert.Equal(t, "tag2", tags[1].Tag) + assert.ElementsMatch(t, []string{"tag1", "tag2"}, []string{tags[0].Tag, tags[1].Tag}) }) t.Run("ownerType returns org", func(t *testing.T) { @@ -161,7 +159,6 @@ func TestClient_Tags(t *testing.T) { tags, err := client.Tags(ctx, host, "test-org-owner", "test-repo") assert.NoError(t, err) assert.Len(t, tags, 2) - assert.Equal(t, "tag1", tags[0].Tag) - assert.Equal(t, "tag2", tags[1].Tag) + assert.ElementsMatch(t, []string{"tag1", "tag2"}, []string{tags[0].Tag, tags[1].Tag}) }) } diff --git a/pkg/client/oci/helpers.go b/pkg/client/oci/helpers.go new file mode 100644 index 00000000..9bd1f909 --- /dev/null +++ b/pkg/client/oci/helpers.go @@ -0,0 +1,25 @@ +package oci + +import ( + "strings" + "time" +) + +const ( + CreatedTimeAnnotation = "org.opencontainers.image.created" + BuildDateAnnotation = "org.label-schema.build-date" +) + +func discoverTimestamp(annotations map[string]string) (timestamp time.Time, err error) { + if t, ok := annotations[CreatedTimeAnnotation]; ok { + timestamp, err = time.Parse(time.RFC3339, + strings.Replace(t, " ", "T", 1), + ) + } else if t, ok = annotations[BuildDateAnnotation]; ok { + timestamp, err = time.Parse(time.RFC3339, + strings.Replace(t, " ", "T", 1), + ) + } + + return timestamp, err +} diff --git a/pkg/client/oci/helpers_test.go b/pkg/client/oci/helpers_test.go new file mode 100644 index 00000000..407919af --- /dev/null +++ b/pkg/client/oci/helpers_test.go @@ -0,0 +1,77 @@ +package oci + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestDiscoverTimestamp(t *testing.T) { + tests := []struct { + name string + annotations map[string]string + expected time.Time + expectErr bool + }{ + { + name: "No annotations", + annotations: map[string]string{}, + expected: time.Time{}, + expectErr: false, + }, + { + name: "Valid CreatedTimeAnnotation", + annotations: map[string]string{ + CreatedTimeAnnotation: "2023-03-15T12:34:56Z", + }, + expected: time.Date(2023, 3, 15, 12, 34, 56, 0, time.UTC), + expectErr: false, + }, + { + name: "Valid BuildDateAnnotation", + annotations: map[string]string{ + BuildDateAnnotation: "2023-03-15T12:34:56Z", + }, + expected: time.Date(2023, 3, 15, 12, 34, 56, 0, time.UTC), + expectErr: false, + }, + { + name: "Invalid CreatedTimeAnnotation format", + annotations: map[string]string{ + CreatedTimeAnnotation: "invalid-date", + }, + expected: time.Time{}, + expectErr: true, + }, + { + name: "Invalid BuildDateAnnotation format", + annotations: map[string]string{ + BuildDateAnnotation: "invalid-date", + }, + expected: time.Time{}, + expectErr: true, + }, + { + name: "Both annotations present, prefer CreatedTimeAnnotation", + annotations: map[string]string{ + CreatedTimeAnnotation: "2023-03-15T12:34:56Z", + BuildDateAnnotation: "2023-01-01T00:00:00Z", + }, + expected: time.Date(2023, 3, 15, 12, 34, 56, 0, time.UTC), + expectErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := discoverTimestamp(tt.annotations) + if tt.expectErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.expected, result) + } + }) + } +} diff --git a/pkg/client/oci/oci.go b/pkg/client/oci/oci.go index e67ed669..3ab51fcb 100644 --- a/pkg/client/oci/oci.go +++ b/pkg/client/oci/oci.go @@ -6,14 +6,20 @@ import ( "net/http" "runtime" "strings" + "sync" + + "github.com/sirupsen/logrus" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/types" "github.com/jetstack/version-checker/pkg/api" ) +var numWorkers = runtime.NumCPU() * 5 + type Options struct { Transporter http.RoundTripper Auth *authn.AuthConfig @@ -29,14 +35,18 @@ func (o *Options) Authorization() (*authn.AuthConfig, error) { // Client is a client for a registry compatible with the OCI Distribution Spec type Client struct { *Options + log *logrus.Entry puller *remote.Puller } +// Ensure that we are an ImageClient +var _ api.ImageClient = (*Client)(nil) + // New returns a new client -func New(opts *Options) (*Client, error) { +func New(opts *Options, log *logrus.Entry) (*Client, error) { pullOpts := []remote.Option{ - remote.WithJobs(runtime.NumCPU()), - remote.WithUserAgent("version-checker"), + remote.WithJobs(numWorkers), + remote.WithUserAgent("version-checker/oci"), remote.WithAuth(opts), } if opts.Transporter != nil { @@ -50,6 +60,7 @@ func New(opts *Options) (*Client, error) { return &Client{ puller: puller, + log: log.WithField("client", "OCI"), Options: opts, }, nil } @@ -70,13 +81,98 @@ func (c *Client) Tags(ctx context.Context, host, repo, image string) ([]api.Imag if err != nil { return nil, fmt.Errorf("listing tags: %w", err) } + c.log.Infof("Collected %v tags..", len(bareTags)) + return c.Manifests(ctx, reg.Repo(repo, image), bareTags) +} - var tags []api.ImageTag - for _, t := range bareTags { - tags = append(tags, api.ImageTag{Tag: t}) +func (c *Client) Manifests(ctx context.Context, repo name.Repository, tags []string) (fulltags []api.ImageTag, err error) { + wg := sync.WaitGroup{} + sem := make(chan struct{}, numWorkers) // limit concurrent fetches + wg.Add(len(tags)) + mu := sync.Mutex{} + + // Lets lookup all the child Manifests (where applicable) + for _, tag := range tags { + go func(repo name.Repository, tag string) { + log := c.log.WithFields(logrus.Fields{"tag": tag, "repo": repo.Name()}) + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + + // Parse the Tag + t, err := name.NewTag(repo.Name() + ":" + tag) + if err != nil { + log.Errorf("parsing Tag: %s", err) + return + } + + // Fetch the manifest + manifest, err := c.puller.Get(ctx, t) + if err != nil { + log.Errorf("getting manifest: %s", err) + return + } + + // Lock when we have the data! + mu.Lock() + defer mu.Unlock() + + ts, err := discoverTimestamp(manifest.Annotations) + if err != nil { + log.Errorf("Unable to discover Timestamp: %s", err) + return + } + + baseTag := api.ImageTag{ + Tag: tag, + Timestamp: ts, + } + + switch manifest.MediaType { + + case types.OCIImageIndex, types.DockerManifestList: + children := []*api.ImageTag{} + imgidx, err := manifest.ImageIndex() + if err != nil { + log.Errorf("getting ImageIndex: %s", err) + return + } + idxman, err := imgidx.IndexManifest() + if err != nil { + log.Errorf("getting IndexManifest: %s", err) + return + } + for _, img := range idxman.Manifests { + + children = append(children, &api.ImageTag{ + Tag: tag, + SHA: img.Digest.String(), + }) + } + baseTag.Children = children + + case types.OCIManifestSchema1, types.DockerManifestSchema2: + img, err := manifest.Image() + if err != nil { + log.Errorf("unable to collect image from manifest: %s", err) + return + } + sha, err := img.Digest() + if err != nil { + log.Errorf("unable to collect digest from manifest: %s", err) + return + } + baseTag.SHA = sha.String() + } + + // Add it to the full tags + fulltags = append(fulltags, baseTag) + }(repo, tag) } + // Wait for everything to complete! + wg.Wait() - return tags, nil + return fulltags, err } // IsHost always returns true because it supports any host @@ -94,7 +190,7 @@ func (c *Client) RepoImageFromPath(path string) (string, string) { } if lenSplit > 1 { - return split[lenSplit-2], split[lenSplit-1] + return strings.Join(split[:len(split)-1], "/"), split[lenSplit-1] } return path, "" diff --git a/pkg/client/oci/oci_test.go b/pkg/client/oci/oci_test.go index f21a2e17..35767107 100644 --- a/pkg/client/oci/oci_test.go +++ b/pkg/client/oci/oci_test.go @@ -3,11 +3,16 @@ package oci import ( "context" "fmt" + "io" + "log" "net/http/httptest" "net/url" "testing" - "github.com/google/go-cmp/cmp" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/registry" "github.com/google/go-containerregistry/pkg/v1/empty" @@ -17,6 +22,8 @@ import ( func TestClientTags(t *testing.T) { ctx := context.Background() + emptySha, err := empty.Image.Digest() + require.NoError(t, err) type testCase struct { repo string @@ -32,23 +39,25 @@ func TestClientTags(t *testing.T) { wantTags: []api.ImageTag{ { Tag: "a", + SHA: emptySha.String(), }, { Tag: "b", + SHA: emptySha.String(), }, { Tag: "c", + SHA: emptySha.String(), }, }, } repo, err := name.NewRepository(fmt.Sprintf("%s/%s/%s", host, tc.repo, tc.img)) - if err != nil { - t.Fatalf("unexpected error parsing repo: %s", err) - } + require.NoError(t, err) + for _, tag := range tc.wantTags { - if err := remote.Write(repo.Tag(tag.Tag), empty.Image); err != nil { - t.Fatalf("unexpected error writing image to tag: %s", err) - } + require.NoError(t, + remote.Write(repo.Tag(tag.Tag), empty.Image), + ) } return tc }, @@ -58,20 +67,21 @@ func TestClientTags(t *testing.T) { wantTags: []api.ImageTag{ { Tag: "a", + SHA: emptySha.String(), }, { Tag: "b", + SHA: emptySha.String(), }, }, } repo, err := name.NewRepository(fmt.Sprintf("%s/%s", host, tc.img)) - if err != nil { - t.Fatalf("unexpected error parsing repo: %s", err) - } + require.NoError(t, err) + for _, tag := range tc.wantTags { - if err := remote.Write(repo.Tag(tag.Tag), empty.Image); err != nil { - t.Fatalf("unexpected error writing image to tag: %s", err) - } + require.NoError(t, + remote.Write(repo.Tag(tag.Tag), empty.Image), + ) } return tc }, @@ -82,17 +92,17 @@ func TestClientTags(t *testing.T) { wantTags: []api.ImageTag{ { Tag: "a", + SHA: emptySha.String(), }, }, } repo, err := name.NewRepository(fmt.Sprintf("%s/%s/%s", host, tc.repo, tc.img)) - if err != nil { - t.Fatalf("unexpected error parsing repo: %s", err) - } + require.NoError(t, err) + for _, tag := range tc.wantTags { - if err := remote.Write(repo.Tag(tag.Tag), empty.Image); err != nil { - t.Fatalf("unexpected error writing image to tag: %s", err) - } + require.NoError(t, + remote.Write(repo.Tag(tag.Tag), empty.Image), + ) } return tc }, @@ -102,18 +112,16 @@ func TestClientTags(t *testing.T) { img: "bar", } repo, err := name.NewRepository(fmt.Sprintf("%s/%s/%s", host, tc.repo, tc.img)) - if err != nil { - t.Fatalf("unexpected error parsing repo: %s", err) - } + require.NoError(t, err) // Write a tag but then delete it so the repository // exists but it has no tags - if err := remote.Write(repo.Tag("latest"), empty.Image); err != nil { - t.Fatalf("unexpected error writing image to tag: %s", err) - } - if err := remote.Delete(repo.Tag("latest")); err != nil { - t.Fatalf("unexpected error writing image to tag: %s", err) - } + require.NoError(t, + remote.Write(repo.Tag("latest"), empty.Image), + ) + require.NoError(t, + remote.Delete(repo.Tag("latest")), + ) return tc }, "should return an error when listing a repository that doesn't exist": func(t *testing.T, host string) *testCase { @@ -129,23 +137,21 @@ func TestClientTags(t *testing.T) { t.Run(testName, func(t *testing.T) { host := setupRegistry(t) - c, err := New(new(Options)) - if err != nil { - t.Fatalf("unexpected error creating client: %s", err) - } + c, err := New(new(Options), logrus.NewEntry(logrus.New())) + require.NoError(t, err) tc := fn(t, host) gotTags, err := c.Tags(ctx, host, tc.repo, tc.img) - if tc.wantErr && err == nil { - t.Errorf("unexpected nil error listing tags") - } - if !tc.wantErr && err != nil { - t.Errorf("unexpected error listing tags: %s", err) - } - if diff := cmp.Diff(tc.wantTags, gotTags); diff != "" { - t.Errorf("unexpected tags:\n%s", diff) - } + + if tc.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + // We don't care about the order - but to ensure that the elements we expect + // exist within the output + assert.ElementsMatch(t, tc.wantTags, gotTags) }) } } @@ -177,27 +183,27 @@ func TestClientRepoImageFromPath(t *testing.T) { }, } - c, err := New(new(Options)) + c, err := New(new(Options), logrus.NewEntry(logrus.New())) if err != nil { t.Fatalf("unexpected error creating client: %s", err) } for name, test := range tests { t.Run(name, func(t *testing.T) { repo, image := c.RepoImageFromPath(test.path) - if repo != test.expRepo && image != test.expImage { - t.Errorf("%s: unexpected repo/image, exp=%s/%s got=%s/%s", - test.path, test.expRepo, test.expImage, repo, image) - } + assert.Equal(t, test.expRepo, repo) + assert.Equal(t, test.expImage, image) }) } } func setupRegistry(t *testing.T) string { - r := httptest.NewServer(registry.New()) + r := httptest.NewServer(registry.New( + registry.Logger(log.New(io.Discard, "", log.LstdFlags)), + registry.WithReferrersSupport(false), + )) t.Cleanup(r.Close) u, err := url.Parse(r.URL) - if err != nil { - t.Fatalf("unexpected error parsing registry url: %s", err) - } + require.NoError(t, err) + return u.Host } diff --git a/pkg/client/quay/api_types.go b/pkg/client/quay/api_types.go new file mode 100644 index 00000000..fc87b091 --- /dev/null +++ b/pkg/client/quay/api_types.go @@ -0,0 +1,35 @@ +package quay + +import ( + "github.com/jetstack/version-checker/pkg/api" +) + +type responseTag struct { + Tags []responseTagItem `json:"tags"` + HasAdditional bool `json:"has_additional"` + Page int `json:"page"` +} + +type responseTagItem struct { + Name string `json:"name"` + ManifestDigest string `json:"manifest_digest"` + LastModified string `json:"last_modified"` + IsManifestList bool `json:"is_manifest_list"` +} + +type responseManifest struct { + Status *int `json:"status,omitempty"` + ManifestData string `json:"manifest_data"` +} + +type responseManifestData struct { + Manifests []responseManifestDataItem `json:"manifests"` +} + +type responseManifestDataItem struct { + Digest string `json:"digest"` + Platform struct { + Architecture api.Architecture `json:"architecture"` + OS api.OS `json:"os"` + } `json:"platform"` +} diff --git a/pkg/client/quay/pager.go b/pkg/client/quay/pager.go index ef3ce7b6..c99a81f7 100644 --- a/pkg/client/quay/pager.go +++ b/pkg/client/quay/pager.go @@ -14,11 +14,11 @@ type pager struct { repo, image string - mu sync.Mutex - wg sync.WaitGroup - tags []api.ImageTag errs []error + wg sync.WaitGroup + + mu sync.Mutex } func (c *Client) newPager(repo, image string) *pager { @@ -55,20 +55,28 @@ func (p *pager) fetchTags(ctx context.Context) error { // fetchTagsPaged will return the image tags from a given page number, as well // as if there are more pages. func (p *pager) fetchTagsPaged(ctx context.Context, page int) (bool, error) { + select { + case <-ctx.Done(): + return false, ctx.Err() + default: + } + url := fmt.Sprintf(tagURL, p.repo, p.image, page) var resp responseTag if err := p.makeRequest(ctx, url, &resp); err != nil { return false, err } + sem := make(chan struct{}, 10) // limit concurrent fetches p.wg.Add(len(resp.Tags)) - // Concurrently fetch all images from a given tag - for i := range resp.Tags { - go func(i int) { + for _, tag := range resp.Tags { + go func(tag responseTagItem) { defer p.wg.Done() + sem <- struct{}{} + defer func() { <-sem }() - imageTags, err := p.fetchImageManifest(ctx, p.repo, p.image, &resp.Tags[i]) + imageTag, err := p.fetchImageManifest(ctx, p.repo, p.image, &tag) p.mu.Lock() defer p.mu.Unlock() @@ -78,8 +86,8 @@ func (p *pager) fetchTagsPaged(ctx context.Context, page int) (bool, error) { return } - p.tags = append(p.tags, imageTags...) - }(i) + p.tags = append(p.tags, *imageTag) + }(tag) } return resp.HasAdditional, nil diff --git a/pkg/client/quay/path_test.go b/pkg/client/quay/path_test.go index 5fff08e1..8cf19b8b 100644 --- a/pkg/client/quay/path_test.go +++ b/pkg/client/quay/path_test.go @@ -1,6 +1,10 @@ package quay -import "testing" +import ( + "testing" + + "github.com/stretchr/testify/assert" +) func TestIsHost(t *testing.T) { tests := map[string]struct { @@ -40,10 +44,9 @@ func TestIsHost(t *testing.T) { handler := new(Client) for name, test := range tests { t.Run(name, func(t *testing.T) { - if isHost := handler.IsHost(test.host); isHost != test.expIs { - t.Errorf("%s: unexpected IsHost, exp=%t got=%t", - test.host, test.expIs, isHost) - } + assert.Equal(t, test.expIs, + handler.IsHost(test.host), + ) }) } } @@ -74,10 +77,8 @@ func TestRepoImage(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { repo, image := handler.RepoImageFromPath(test.path) - if repo != test.expRepo && image != test.expImage { - t.Errorf("%s: unexpected repo/image, exp=%s/%s got=%s/%s", - test.path, test.expRepo, test.expImage, repo, image) - } + assert.Equal(t, test.expRepo, repo) + assert.Equal(t, test.expImage, image) }) } } diff --git a/pkg/client/quay/quay.go b/pkg/client/quay/quay.go index 7130084a..58dcd395 100644 --- a/pkg/client/quay/quay.go +++ b/pkg/client/quay/quay.go @@ -20,8 +20,8 @@ const ( ) type Options struct { - Token string Transporter http.RoundTripper + Token string } type Client struct { @@ -29,35 +29,8 @@ type Client struct { Options } -type responseTag struct { - Tags []responseTagItem `json:"tags"` - HasAdditional bool `json:"has_additional"` - Page int `json:"page"` -} - -type responseTagItem struct { - Name string `json:"name"` - ManifestDigest string `json:"manifest_digest"` - LastModified string `json:"last_modified"` - IsManifestList bool `json:"is_manifest_list"` -} - -type responseManifest struct { - ManifestData string `json:"manifest_data"` - Status *int `json:"status,omitempty"` -} - -type responseManifestData struct { - Manifests []responseManifestDataItem `json:"manifests"` -} - -type responseManifestDataItem struct { - Digest string `json:"digest"` - Platform struct { - Architecture api.Architecture `json:"architecture"` - OS api.OS `json:"os"` - } `json:"platform"` -} +// Ensure that we are an ImageClient +var _ api.ImageClient = (*Client)(nil) func New(opts Options, log *logrus.Entry) *Client { client := retryablehttp.NewClient() @@ -89,60 +62,58 @@ func (c *Client) Tags(ctx context.Context, _, repo, image string) ([]api.ImageTa } // fetchImageManifest will lookup all manifests for a tag, if it is a list. -func (c *Client) fetchImageManifest(ctx context.Context, repo, image string, tag *responseTagItem) ([]api.ImageTag, error) { +func (c *Client) fetchImageManifest(ctx context.Context, repo, image string, tag *responseTagItem) (*api.ImageTag, error) { timestamp, err := time.Parse(time.RFC1123Z, tag.LastModified) if err != nil { return nil, err } + iTag := &api.ImageTag{ + Tag: tag.Name, + SHA: tag.ManifestDigest, + Timestamp: timestamp, + OS: "", + Architecture: "", + } + // If a multi-arch image, call manifest endpoint if tag.IsManifestList { url := fmt.Sprintf(manifestURL, repo, image, tag.ManifestDigest) - tags, err := c.callManifests(ctx, timestamp, tag.Name, url) + err := c.callManifests(ctx, timestamp, iTag, url) if err != nil { return nil, err } - return tags, nil + return iTag, nil } // Fallback to not using multi-arch image + iTag.OS, iTag.Architecture = util.OSArchFromTag(tag.Name) - os, arch := util.OSArchFromTag(tag.Name) - - return []api.ImageTag{ - { - Tag: tag.Name, - SHA: tag.ManifestDigest, - Timestamp: timestamp, - OS: os, - Architecture: arch, - }, - }, nil + return iTag, nil } // callManifests endpoint on the tags image manifest. -func (c *Client) callManifests(ctx context.Context, timestamp time.Time, tag, url string) ([]api.ImageTag, error) { +func (c *Client) callManifests(ctx context.Context, timestamp time.Time, tag *api.ImageTag, url string) error { var manifestResp responseManifest if err := c.makeRequest(ctx, url, &manifestResp); err != nil { - return nil, err + return err } // Got error on this manifest, ignore if manifestResp.Status != nil { - return nil, nil + return nil } var manifestData responseManifestData if err := json.Unmarshal([]byte(manifestResp.ManifestData), &manifestData); err != nil { - return nil, fmt.Errorf("failed to unmarshal manifest data %s: %#+v: %s", - tag, manifestResp, err) + return fmt.Errorf("failed to unmarshal manifest data %s: %#+v: %s", + tag.Tag, manifestResp, err) } - var tags []api.ImageTag for _, manifest := range manifestData.Manifests { - tags = append(tags, api.ImageTag{ - Tag: tag, + tag.Children = append(tag.Children, &api.ImageTag{ + Tag: tag.Tag, SHA: manifest.Digest, Timestamp: timestamp, Architecture: manifest.Platform.Architecture, @@ -150,7 +121,7 @@ func (c *Client) callManifests(ctx context.Context, timestamp time.Time, tag, ur }) } - return tags, nil + return nil } // makeRequest will make a call and write the response to the object. diff --git a/pkg/client/selfhosted/api_types.go b/pkg/client/selfhosted/api_types.go index af205d2c..66ef4024 100644 --- a/pkg/client/selfhosted/api_types.go +++ b/pkg/client/selfhosted/api_types.go @@ -1,6 +1,7 @@ package selfhosted import ( + "encoding/json" "time" "github.com/jetstack/version-checker/pkg/api" @@ -15,15 +16,60 @@ type TagResponse struct { } type ManifestResponse struct { - Digest string + Digest string `json:"digest,omitempty"` Architecture api.Architecture `json:"architecture"` History []History `json:"history"` } +type ManafestListResponse struct { + Manifests []ManifestResponse `json:"manifests"` +} + type History struct { - V1Compatibility string `json:"v1Compatibility"` + V1Compatibility V1CompatibilityWrapper `json:"v1Compatibility"` } type V1Compatibility struct { Created time.Time `json:"created,omitempty"` } + +type V1CompatibilityWrapper struct { + V1Compatibility +} + +func (v *V1CompatibilityWrapper) UnmarshalJSON(b []byte) error { + var raw string + if err := json.Unmarshal(b, &raw); err != nil { + return err + } + return json.Unmarshal([]byte(raw), &v.V1Compatibility) +} + +func (v V1CompatibilityWrapper) MarshalJSON() ([]byte, error) { + marshaled, err := json.Marshal(v.V1Compatibility) + if err != nil { + return nil, err + } + return json.Marshal(string(marshaled)) // ← Double encode: inner to string +} + +type ErrorResponse struct { + Errors []ErrorType `json:"errors"` +} + +type ErrorType struct { + Code string `json:"code"` + Message string `json:"message"` +} + +type V2ManifestListResponse struct { + SchemaVersion int `json:"schemaVersion"` + MediaType string `json:"mediaType"` + Manifests []V2ManifestListEntry `json:"manifests"` +} + +type V2ManifestListEntry struct { + Digest string `json:"digest"` + MediaType string `json:"mediaType"` + Platform api.Platform `json:"platform"` +} diff --git a/pkg/client/selfhosted/path.go b/pkg/client/selfhosted/path.go index 3e444c00..9d99b049 100644 --- a/pkg/client/selfhosted/path.go +++ b/pkg/client/selfhosted/path.go @@ -13,6 +13,9 @@ const ( ) func (c *Client) IsHost(host string) bool { + if c.hostRegex == nil { + return c.Host == host + } return c.hostRegex.MatchString(host) } diff --git a/pkg/client/selfhosted/path_test.go b/pkg/client/selfhosted/path_test.go index 6824427f..0df5684a 100644 --- a/pkg/client/selfhosted/path_test.go +++ b/pkg/client/selfhosted/path_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" ) func TestIsHost(t *testing.T) { @@ -77,12 +78,28 @@ func TestIsHost(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { - if isHost := handler.IsHost(test.host); isHost != test.expIs { - t.Errorf("%s: unexpected IsHost, exp=%t got=%t", - test.host, test.expIs, isHost) - } + assert.Equal(t, test.expIs, + handler.IsHost(test.host), + ) + assert.Equal(t, test.expIs, handler.IsHost(test.host)) }) } + + // t.Run("No Options set on client", func(t *testing.T) { + // handler, err := New(context.TODO(), logrus.NewEntry(logrus.New()), &Options{}) + // require.NoError(t, err) + + // assert.NotPanics(t, func() { handler.IsHost("example.com") }) + // }) + + // t.Run("No Options set on client", func(t *testing.T) { + + // handler := &Client{Options: &Options{Host: "abc"}} + // require.NoError(t, err) + + // assert.NotPanics(t, func() { handler.IsHost("example.com") }) + // assert.True(t, handler.IsHost("abc")) + // }) } func TestRepoImage(t *testing.T) { diff --git a/pkg/client/selfhosted/selfhosted.go b/pkg/client/selfhosted/selfhosted.go index 47982684..abfb2b40 100644 --- a/pkg/client/selfhosted/selfhosted.go +++ b/pkg/client/selfhosted/selfhosted.go @@ -24,6 +24,9 @@ import ( "github.com/jetstack/version-checker/pkg/client/util" ) +// Ensure that we are an ImageClient +var _ api.ImageClient = (*Client)(nil) + const ( // {host}/v2/{repo/image}/tags/list?n=500 tagsPath = "%s/v2/%s/tags/list?n=500" @@ -33,21 +36,22 @@ const ( defaultTokenPath = "/v2/token" // HTTP headers to request API version - dockerAPIv1Header = "application/vnd.docker.distribution.manifest.v1+json" - dockerAPIv2Header = "application/vnd.docker.distribution.manifest.v2+json" + dockerAPIv1Header = "application/vnd.docker.distribution.manifest.v1+json" + dockerAPIv2Header = "application/vnd.docker.distribution.manifest.v2+json" + dockerAPIv2ManifestList = "application/vnd.docker.distribution.manifest.list.v2+json" ) type Options struct { - Host string - Username string - Password string - Bearer string - TokenPath string - Insecure bool - CAPath string Transporter http.RoundTripper -} + Host string + Username string + Password string + Bearer string + TokenPath string + CAPath string + Insecure bool +} type Client struct { *http.Client *Options @@ -169,7 +173,7 @@ func (c *Client) Tags(ctx context.Context, host, repo, image string) ([]api.Imag return nil, err } - var tags []api.ImageTag + tags := map[string]api.ImageTag{} for _, tag := range tagResponse.Tags { manifestURL := fmt.Sprintf(manifestPath, host, path, tag) @@ -187,20 +191,16 @@ func (c *Client) Tags(ctx context.Context, host, repo, image string) ([]api.Imag var timestamp time.Time for _, v1History := range manifestResponse.History { - data := V1Compatibility{} - if err := json.Unmarshal([]byte(v1History.V1Compatibility), &data); err != nil { - return nil, err - } - - if !data.Created.IsZero() { - timestamp = data.Created + if !v1History.V1Compatibility.Created.IsZero() { + timestamp = v1History.V1Compatibility.Created // Each layer has its own created timestamp. We just want a general reference. // Take the first and step out the loop break } } - header, err := c.doRequest(ctx, manifestURL, dockerAPIv2Header, new(ManifestResponse)) + var manifestListResponse V2ManifestListResponse + header, err := c.doRequest(ctx, manifestURL, strings.Join([]string{dockerAPIv2Header, dockerAPIv2ManifestList}, ","), &manifestListResponse) if httpErr, ok := selfhostederrors.IsHTTPError(err); ok { c.log.Errorf("%s: failed to get manifest sha response for tag, skipping (%d): %s", manifestURL, httpErr.StatusCode, httpErr.Body) @@ -210,15 +210,28 @@ func (c *Client) Tags(ctx context.Context, host, repo, image string) ([]api.Imag return nil, err } - tags = append(tags, api.ImageTag{ + // Lets set as much of the current as we know + current := api.ImageTag{ Tag: tag, SHA: header.Get("Docker-Content-Digest"), Timestamp: timestamp, - Architecture: manifestResponse.Architecture, - }) - } + Architecture: api.Architecture(manifestResponse.Architecture), + } - return tags, nil + util.FindParentTags(tags, tag, ¤t) + + for _, manifest := range manifestListResponse.Manifests { + + // If we didn't get a SHA from the inital call, + // lets set it from the manifestList + if current.SHA != "" && manifest.Digest != "" { + current.SHA = manifest.Digest + } + + util.FindParentTags(tags, tag, ¤t) + } + } + return util.TagMaptoList(tags), nil } func (c *Client) doRequest(ctx context.Context, url, header string, obj interface{}) (http.Header, error) { @@ -227,6 +240,7 @@ func (c *Client) doRequest(ctx context.Context, url, header string, obj interfac if err != nil { return nil, err } + req.Header.Set("User-Agent", "version-checker/selfhosted") req = req.WithContext(ctx) if len(c.Bearer) > 0 { @@ -256,7 +270,7 @@ func (c *Client) doRequest(ctx context.Context, url, header string, obj interfac } if err := json.Unmarshal(body, obj); err != nil { - return nil, fmt.Errorf("unexpected %s response: %s", url, body) + return nil, fmt.Errorf("unexpected %s response: %s - %w", url, body, err) } return resp.Header, nil diff --git a/pkg/client/selfhosted/selfhosted_test.go b/pkg/client/selfhosted/selfhosted_test.go index 223e6e6b..3aa30bb4 100644 --- a/pkg/client/selfhosted/selfhosted_test.go +++ b/pkg/client/selfhosted/selfhosted_test.go @@ -3,16 +3,20 @@ package selfhosted import ( "context" "encoding/base64" + "encoding/json" "errors" "fmt" "net/http" "net/http/httptest" "net/url" "os" + "strings" "testing" + "time" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/jetstack/version-checker/pkg/api" selfhostederrors "github.com/jetstack/version-checker/pkg/client/selfhosted/errors" @@ -55,6 +59,17 @@ func TestNew(t *testing.T) { assert.Contains(t, err.Error(), "failed parsing url") }) + // t.Run("Error on missing host", func(t *testing.T) { + // opts := &Options{ + // Host: "", + // CAPath: "invalid/path", + // } + // client, err := New(ctx, log, opts) + // assert.Nil(t, client) + // assert.Error(t, err) + // assert.Contains(t, err.Error(), "host cannot be empty") + // }) + t.Run("error on username/password and bearer token both specified", func(t *testing.T) { opts := &Options{ Host: "https://testregistry.com", @@ -126,17 +141,70 @@ func TestTags(t *testing.T) { } server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // l.Infof("Got request: %v", r) switch r.URL.Path { case "/v2/repo/image/tags/list": w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(`{"tags":["v1.0.0","v2.0.0"]}`)) + _ = json.NewEncoder(w).Encode(TagResponse{Tags: []string{"v1.0.0", "v2.0.0"}}) + case "/v2/repo/image/manifests/v1.0.0": w.Header().Add("Docker-Content-Digest", "sha256:abcdef") w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(`{"architecture":"amd64","history":[{"v1Compatibility":"{\"created\":\"2023-08-27T12:00:00Z\"}"}]}`)) + _ = json.NewEncoder(w).Encode(ManifestResponse{ + Architecture: api.Architecture("amd64"), + History: []History{ + { + V1Compatibility: V1CompatibilityWrapper{ + V1Compatibility: V1Compatibility{Created: time.Now()}, + }, + }, + }, + }) + case "/v2/repo/image/manifests/v2.0.0": w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(`{}`)) // Write some blank content + + // This image is a manifest List + case "/v2/repo/multiimage/tags/list": + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(TagResponse{Tags: []string{"v2.2.0"}}) + + case "/v2/repo/multiimage/manifests/v2.2.0": + acpt := r.Header.Get("Accept") + log.Warnf("Got following request: %v", acpt) + switch acpt { + // If we have multiple formats... + case strings.Join([]string{dockerAPIv2Header, dockerAPIv2ManifestList}, ","): + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(V2ManifestListResponse{ + Manifests: []V2ManifestListEntry{ + {Digest: "asjhfvbasjhbfsaj", Platform: api.Platform{OS: api.OS("Linux"), Architecture: api.Architecture("arm64")}}, + }, + }) + + // Docker V1 API + case dockerAPIv1Header: + w.WriteHeader(http.StatusOK) + w.Header().Add("Docker-Content-Digest", "sha265:asgjnaskjgbsajgsa") + _, _ = w.Write([]byte(`{}`)) // Write some blank content + + // Docker V2 Header... + case dockerAPIv2Header: + w.WriteHeader(http.StatusNotFound) + _ = json.NewEncoder(w).Encode(ErrorResponse{Errors: []ErrorType{ + { + Code: "MANIFEST_UNKNOWN", + Message: `Manifest has media type "application/vnd.docker.distribution.manifest.list.v2+json" but client accepts ["application/vnd.docker.distribution.manifest.v1+json"]`, + }, + }}) + + // ManifestList ONLY + case dockerAPIv2ManifestList: + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{}`)) // Write some blank content + + } } })) defer server.Close() @@ -144,14 +212,24 @@ func TestTags(t *testing.T) { h, err := url.Parse(server.URL) assert.NoError(t, err) - tags, err := client.Tags(ctx, h.Host, "repo", "image") + t.Run("Standard Single Arch Image", func(t *testing.T) { + tags, err := client.Tags(ctx, h.Host, "repo", "image") + require.NoError(t, err) + require.Len(t, tags, 2) - assert.NoError(t, err) - assert.Len(t, tags, 2) - assert.Equal(t, "v1.0.0", tags[0].Tag) - assert.Equal(t, api.Architecture("amd64"), tags[0].Architecture) - assert.Equal(t, "sha256:abcdef", tags[0].SHA) - assert.Equal(t, "v2.0.0", tags[1].Tag) + // We don't care of the order, we just want to make sure we have the tags + assert.ElementsMatch(t, []string{"v1.0.0", "v2.0.0"}, []string{tags[0].Tag, tags[1].Tag}) + assert.Equal(t, api.Architecture("amd64"), tags[0].Architecture) + assert.Equal(t, "sha256:abcdef", tags[0].SHA) + }) + + t.Run("MultiArch ManifestList v2.2", func(t *testing.T) { + tags, err := client.Tags(ctx, h.Host, "repo", "multiimage") + + assert.NoError(t, err) + require.Len(t, tags, 1) + assert.Equal(t, "v2.2.0", tags[0].Tag) + }) }) t.Run("error fetching tags", func(t *testing.T) { diff --git a/pkg/client/util/http_backoff.go b/pkg/client/util/http_backoff.go new file mode 100644 index 00000000..465c5210 --- /dev/null +++ b/pkg/client/util/http_backoff.go @@ -0,0 +1,19 @@ +package util + +import ( + "net/http" + "time" + + "github.com/hashicorp/go-retryablehttp" +) + +// This is a custom Backoff that enforces the Max wait duration. +// If the sleep is greater we refuse to sleep at all +func HTTPBackOff(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration { + sleep := retryablehttp.DefaultBackoff(min, max, attemptNum, resp) + if sleep.Abs() <= max { + return sleep.Abs() + } + + return max.Abs() +} diff --git a/pkg/client/util/http_backoff_limiter.go b/pkg/client/util/http_backoff_limiter.go new file mode 100644 index 00000000..02b50b8c --- /dev/null +++ b/pkg/client/util/http_backoff_limiter.go @@ -0,0 +1,49 @@ +package util + +import ( + "fmt" + "net/http" + "time" + + "github.com/sirupsen/logrus" + + "github.com/hashicorp/go-retryablehttp" + "golang.org/x/time/rate" +) + +func RateLimitedBackoffLimiter( + logger *logrus.Entry, + limiter *rate.Limiter, + maxWait time.Duration, +) retryablehttp.Backoff { + return func(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration { + + defaultDelay := retryablehttp.DefaultBackoff(min, max, attemptNum, resp) + // Reserve first to introspect delay + res := limiter.Reserve() + if !res.OK() { + logger.Error(fmt.Errorf("rate limit exceeded"), "Cannot make request") + return maxWait // fallback + } + rateDelay := res.Delay() + + // Choose the larger of the two delays (rate limit or default backoff) + delay := defaultDelay + if rateDelay > delay { + delay = rateDelay + } + + if delay > maxWait { + res.Cancel() + logger.WithFields(logrus.Fields{ + "attempt": attemptNum, "wait": delay, "maxWait": maxWait, + }).Info("Wait time too long, using max wait instead") + return maxWait + } + + logger.WithFields(logrus.Fields{ + "attempt": attemptNum, "wait": delay, + }).Info("Waiting due to rate limit") + return delay + } +} diff --git a/pkg/client/util/http_backoff_limiter_test.go b/pkg/client/util/http_backoff_limiter_test.go new file mode 100644 index 00000000..1a553c31 --- /dev/null +++ b/pkg/client/util/http_backoff_limiter_test.go @@ -0,0 +1,65 @@ +package util + +import ( + "testing" + "time" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "golang.org/x/time/rate" +) + +func TestRateLimitedBackoffLimiter(t *testing.T) { + logger := logrus.NewEntry(logrus.New()) + maxWait := 5 * time.Second + + t.Run("default backoff delay is used when rate limiter allows immediate execution", func(t *testing.T) { + limiter := rate.NewLimiter(rate.Every(1*time.Millisecond), 1) + backoff := RateLimitedBackoffLimiter(logger, limiter, maxWait) + + min := 100 * time.Millisecond + max := 200 * time.Millisecond + attemptNum := 1 + + delay := backoff(min, max, attemptNum, nil) + assert.GreaterOrEqual(t, delay, min) + assert.LessOrEqual(t, delay, max) + }) + + t.Run("rate limiter delay is used when it exceeds default backoff", func(t *testing.T) { + limiter := rate.NewLimiter(rate.Every(500*time.Millisecond), 1) + backoff := RateLimitedBackoffLimiter(logger, limiter, maxWait) + + min := 100 * time.Millisecond + max := 500 * time.Millisecond + attemptNum := 3 + + delay := backoff(min, max, attemptNum, nil) + assert.GreaterOrEqual(t, delay, 500*time.Millisecond) + assert.LessOrEqual(t, delay, maxWait) + }) + + t.Run("maxWait is used when delay exceeds maxWait", func(t *testing.T) { + limiter := rate.NewLimiter(rate.Every(10*time.Second), 1) + backoff := RateLimitedBackoffLimiter(logger, limiter, maxWait) + + min := maxWait - time.Second + max := maxWait + attemptNum := 3 + + delay := backoff(min, max, attemptNum, nil) + assert.Equal(t, maxWait, delay) + }) + + t.Run("rate limiter reservation fails", func(t *testing.T) { + limiter := rate.NewLimiter(rate.Every(1*time.Second), 0) // No tokens available + backoff := RateLimitedBackoffLimiter(logger, limiter, maxWait) + + min := 100 * time.Millisecond + max := 500 * time.Millisecond + attemptNum := 3 + + delay := backoff(min, max, attemptNum, nil) + assert.Equal(t, maxWait, delay) + }) +} diff --git a/pkg/client/util/util.go b/pkg/client/util/util.go index 018a2c62..b29e2f98 100644 --- a/pkg/client/util/util.go +++ b/pkg/client/util/util.go @@ -46,12 +46,13 @@ func JoinRepoImage(repo, image string) string { } // Attempt to determine the OS and Arch, given a tag name. -func OSArchFromTag(tag string) (api.OS, api.Architecture) { - var ( - os api.OS - arch api.Architecture - split = strings.Split(tag, "-") - ) +func OSArchFromTag(tag string) (os api.OS, arch api.Architecture) { + split := strings.Split(tag, "-") + + // If we don't have >3 splits, then we may not have + if len(split) == 2 { + os = api.OS("linux") + } for _, s := range split { ss := strings.ToLower(s) @@ -71,3 +72,22 @@ func OSArchFromTag(tag string) (api.OS, api.Architecture) { return os, arch } + +func TagMaptoList(tags map[string]api.ImageTag) []api.ImageTag { + taglist := make([]api.ImageTag, 0, len(tags)) + for _, tag := range tags { + taglist = append(taglist, tag) + } + return taglist +} + +func FindParentTags(tags map[string]api.ImageTag, tag string, current *api.ImageTag) { + // Already exists — add as child + if parent, exists := tags[tag]; exists { + parent.Children = append(parent.Children, current) + tags[tag] = parent + } else { + // First occurrence — assign as root + tags[tag] = *current + } +} diff --git a/pkg/client/util/util_test.go b/pkg/client/util/util_test.go index 2955055a..39d0bd9f 100644 --- a/pkg/client/util/util_test.go +++ b/pkg/client/util/util_test.go @@ -2,6 +2,10 @@ package util import ( "testing" + + "github.com/stretchr/testify/assert" + + "github.com/jetstack/version-checker/pkg/api" ) func TestJoinRepoImage(t *testing.T) { @@ -45,3 +49,89 @@ func TestJoinRepoImage(t *testing.T) { }) } } + +func TestOSArchFromTag(t *testing.T) { + tests := map[string]struct { + tag string + expOS api.OS + expArch api.Architecture + }{ + "empty tag should return empty OS and Arch": { + tag: "", + expOS: "", + expArch: "", + }, + "tag with only OS should return correct OS and empty Arch": { + tag: "v1.0.0-linux", + expOS: "linux", + expArch: "", + }, + "tag with only Arch should return linux OS and correct Arch": { + tag: "v1.0.0-amd64", + expOS: "linux", + expArch: "amd64", + }, + "tag with OS and Arch should return both correctly": { + tag: "v1.0.0-linux-amd64", + expOS: "linux", + expArch: "amd64", + }, + "tag with unknown OS and Arch should return empty OS and Arch": { + tag: "v1.0.0-os-unknown-arch", + expOS: "", + expArch: "", + }, + "tag with mixed case OS and Arch should return correct OS and Arch": { + tag: "v1.0.0-Linux-AMD64", + expOS: "linux", + expArch: "amd64", + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + os, arch := OSArchFromTag(test.tag) + + assert.Equal(t, os, test.expOS) + assert.Equal(t, arch, test.expArch) + }) + } +} +func TestTagMaptoList(t *testing.T) { + tests := map[string]struct { + tags map[string]api.ImageTag + expList []api.ImageTag + }{ + "empty map should return empty list": { + tags: map[string]api.ImageTag{}, + expList: []api.ImageTag{}, + }, + "single entry map should return single element list": { + tags: map[string]api.ImageTag{ + "v1.0.0": {Tag: "v1.0.0"}, + }, + expList: []api.ImageTag{ + {Tag: "v1.0.0"}, + }, + }, + "multiple entry map should return list with all elements": { + tags: map[string]api.ImageTag{ + "v1.0.0": {Tag: "v1.0.0"}, + "v1.1.0": {Tag: "v1.1.0"}, + "v2.0.0": {Tag: "v2.0.0"}, + }, + expList: []api.ImageTag{ + {Tag: "v1.0.0"}, + {Tag: "v1.1.0"}, + {Tag: "v2.0.0"}, + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + result := TagMaptoList(test.tags) + assert.ElementsMatch(t, result, test.expList) + }) + } +} diff --git a/pkg/controller/checker/checker.go b/pkg/controller/checker/checker.go index a80f81ba..387d2ad1 100644 --- a/pkg/controller/checker/checker.go +++ b/pkg/controller/checker/checker.go @@ -20,8 +20,8 @@ type Checker struct { type Result struct { CurrentVersion string LatestVersion string - IsLatest bool ImageURL string + IsLatest bool } func New(search search.Searcher) *Checker { @@ -145,11 +145,24 @@ func (c *Checker) isLatestSemver(ctx context.Context, imageURL, currentSHA strin isLatest = true } - // If using the same image version, but the SHA has been updated upstream, - // make not latest - if currentImage.Equal(latestImageV) && currentSHA != latestImage.SHA && latestImage.SHA != "" { + // If using the same image version, + // but the SHA has been updated upstream, + // mark not latest + if currentImage.Equal(latestImageV) && + !latestImage.MatchesSHA(currentSHA) { isLatest = false - latestImage.Tag = fmt.Sprintf("%s@%s", latestImage.Tag, latestImage.SHA) + if latestImage.SHA != "" { + // Add the SHA as a prefix to identify that it has been updated! + latestImage.Tag = fmt.Sprintf("%s@%s", latestImage.Tag, latestImage.SHA) + } else { + for _, child := range latestImage.Children { + // Take first child's SHA for latest image tag + if child.SHA != currentSHA { + latestImage.Tag = fmt.Sprintf("%s@%s", latestImage.Tag, child.SHA) + break + } + } + } } return latestImage, isLatest, nil @@ -162,10 +175,26 @@ func (c *Checker) isLatestSHA(ctx context.Context, imageURL, currentSHA string, return nil, err } - isLatest := latestImage.SHA == currentSHA - latestVersion := latestImage.SHA - if len(latestImage.Tag) > 0 { - latestVersion = fmt.Sprintf("%s@%s", latestImage.Tag, latestImage.SHA) + var ( + isLatest bool + latestVersion string + ) + + if latestImage.SHA != "" { + isLatest = latestImage.SHA == currentSHA + latestVersion = latestImage.SHA + } + + for _, child := range latestImage.Children { + if child.SHA == currentSHA { + isLatest = true + latestVersion = child.SHA + break + } + } + + if len(latestImage.Tag) > 0 && latestVersion != "" { + latestVersion = fmt.Sprintf("%s@%s", latestImage.Tag, latestVersion) } return &Result{ diff --git a/pkg/controller/checker/checker_test.go b/pkg/controller/checker/checker_test.go index ac5dadd6..b5d1f598 100644 --- a/pkg/controller/checker/checker_test.go +++ b/pkg/controller/checker/checker_test.go @@ -8,6 +8,9 @@ import ( "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/jetstack/version-checker/pkg/api" "github.com/jetstack/version-checker/pkg/controller/internal/fake/search" "github.com/jetstack/version-checker/pkg/version/semver" @@ -58,6 +61,54 @@ func TestContainer(t *testing.T) { IsLatest: true, }, }, + "if v0.2.0 is latest version, but sha is in a child, then latest": { + statusSHA: "localhost:5000/version-checker@sha:123", + imageURL: "localhost:5000/version-checker:v0.2.0", + opts: new(api.Options), + searchResp: &api.ImageTag{ + Tag: "v0.2.0", + SHA: "sha:abc1234", + Children: []*api.ImageTag{{SHA: "sha:123"}}, + }, + expResult: &Result{ + CurrentVersion: "v0.2.0", + LatestVersion: "v0.2.0", + ImageURL: "localhost:5000/version-checker", + IsLatest: true, + }, + }, + "if v0.2.0 is latest version, but sha is not in cache, then not latest": { + statusSHA: "localhost:5000/version-checker@sha:123", + imageURL: "localhost:5000/version-checker:v0.2.0", + opts: new(api.Options), + searchResp: &api.ImageTag{ + Tag: "v0.2.0", + SHA: "", + Children: []*api.ImageTag{{SHA: "sha:789"}}, + }, + expResult: &Result{ + CurrentVersion: "v0.2.0@sha:123", + LatestVersion: "v0.2.0@sha:789", + ImageURL: "localhost:5000/version-checker", + IsLatest: false, + }, + }, + "if v0.2.0 is latest version, but sha is not in cache, and multiple possible shas, then not latest": { + statusSHA: "localhost:5000/version-checker@123", + imageURL: "localhost:5000/version-checker:v0.2.0", + opts: new(api.Options), + searchResp: &api.ImageTag{ + Tag: "v0.2.0", + SHA: "", + Children: []*api.ImageTag{{SHA: "789"}, {SHA: "sha:987"}}, + }, + expResult: &Result{ + CurrentVersion: "v0.2.0@123", + LatestVersion: "v0.2.0@789", + ImageURL: "localhost:5000/version-checker", + IsLatest: false, + }, + }, "if v0.2.0@sha:123 is wrong sha, then not latest": { statusSHA: "localhost:5000/version-checker@sha:123", imageURL: "localhost:5000/version-checker:v0.2.0@sha:123", @@ -280,14 +331,8 @@ func TestContainer(t *testing.T) { } result, err := checker.Container(context.TODO(), logrus.NewEntry(logrus.New()), pod, container, test.opts) - if err != nil { - t.Errorf("unexpected error: %s", err) - } - - if !reflect.DeepEqual(test.expResult, result) { - t.Errorf("got unexpected result, exp=%#+v got=%#+v", - test.expResult, result) - } + require.NoError(t, err) + assert.Exactly(t, test.expResult, result) }) } } @@ -500,20 +545,21 @@ func TestIsLatestSHA(t *testing.T) { searchResp *api.ImageTag expResult *Result }{ - "if SHA not eqaual, then should be not equal": { + "if SHA not equal, then should be not equal": { imageURL: "docker.io", currentSHA: "123", searchResp: &api.ImageTag{ SHA: "456", + Tag: "foo", }, expResult: &Result{ CurrentVersion: "123", - LatestVersion: "456", + LatestVersion: "foo@456", IsLatest: false, ImageURL: "docker.io", }, }, - "if SHA eqaual, then should be equal": { + "if SHA equal, then should be equal": { imageURL: "docker.io", currentSHA: "123", searchResp: &api.ImageTag{ @@ -526,6 +572,60 @@ func TestIsLatestSHA(t *testing.T) { ImageURL: "docker.io", }, }, + "if child SHA equal, then should be equal": { + imageURL: "docker.io", + currentSHA: "123", + searchResp: &api.ImageTag{ + SHA: "456", + Tag: "foo", + Children: []*api.ImageTag{ + { + SHA: "123", + }, + }, + }, + expResult: &Result{ + CurrentVersion: "123", + LatestVersion: "foo@123", + IsLatest: true, + ImageURL: "docker.io", + }, + }, + "if child SHA equal, and parent SHA empty, then should be equal": { + imageURL: "docker.io", + currentSHA: "123", + searchResp: &api.ImageTag{ + SHA: "", + Tag: "foo", + Children: []*api.ImageTag{ + {SHA: "123"}, + {SHA: "456"}, + }, + }, + expResult: &Result{ + CurrentVersion: "123", + LatestVersion: "foo@123", + IsLatest: true, + ImageURL: "docker.io", + }, + }, + "if child SHA not equal, and parent SHA empty, then should not be equal": { + imageURL: "docker.io", + currentSHA: "123", + searchResp: &api.ImageTag{ + SHA: "", + Children: []*api.ImageTag{ + {SHA: "456"}, + {SHA: "789"}, + }, + }, + expResult: &Result{ + CurrentVersion: "123", + LatestVersion: "", + IsLatest: false, + ImageURL: "docker.io", + }, + }, } for name, test := range tests { diff --git a/pkg/controller/options/options.go b/pkg/controller/options/options.go index f5264450..20ae0798 100644 --- a/pkg/controller/options/options.go +++ b/pkg/controller/options/options.go @@ -65,12 +65,14 @@ func (b *Builder) Options(name string) (*api.Options, error) { return &opts, nil } + func (b *Builder) handleSHAOption(name string, opts *api.Options, setNonSha *bool, errs *[]string) error { if useSHA, ok := b.ans[b.index(name, api.UseSHAAnnotationKey)]; ok && useSHA == "true" { opts.UseSHA = true } return nil } + func (b *Builder) handleSHAToTagOption(name string, opts *api.Options, setNonSha *bool, errs *[]string) error { if ResolveSHAToTags, ok := b.ans[b.index(name, api.ResolveSHAToTagsKey)]; ok && ResolveSHAToTags == "true" { opts.ResolveSHAToTags = true diff --git a/pkg/controller/pod_controller.go b/pkg/controller/pod_controller.go index 873260ea..91fb8e9d 100644 --- a/pkg/controller/pod_controller.go +++ b/pkg/controller/pod_controller.go @@ -12,6 +12,7 @@ import ( k8sclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" "github.com/jetstack/version-checker/pkg/client" @@ -43,21 +44,21 @@ func NewPodReconciler( imageClient *client.Client, kubeClient k8sclient.Client, log *logrus.Entry, + requeueDuration time.Duration, defaultTestAll bool, ) *PodReconciler { log = log.WithField("controller", "pod") versionGetter := version.New(log, imageClient, cacheTimeout) search := search.New(log, cacheTimeout, versionGetter) - c := &PodReconciler{ - Log: log, - Client: kubeClient, - Metrics: metrics, - VersionChecker: checker.New(search), - defaultTestAll: defaultTestAll, + return &PodReconciler{ + Log: log, + Client: kubeClient, + Metrics: metrics, + VersionChecker: checker.New(search), + defaultTestAll: defaultTestAll, + RequeueDuration: requeueDuration, } - - return c } // Reconcile is triggered whenever a watched object changes. @@ -77,7 +78,7 @@ func (r *PodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R return ctrl.Result{}, err } - // Perform the version check (your sync logic) + // Perform the version check if err := r.sync(ctx, pod); err != nil { log.Error(err, "Failed to process pod") // Requeue after some time in case of failure @@ -93,10 +94,22 @@ func (r *PodReconciler) SetupWithManager(mgr ctrl.Manager) error { LeaderElect := false return ctrl.NewControllerManagedBy(mgr). For(&corev1.Pod{}, builder.OnlyMetadata). - WithOptions(controller.Options{MaxConcurrentReconciles: numWorkers, NeedLeaderElection: &LeaderElect}). + WithOptions(controller.Options{ + MaxConcurrentReconciles: numWorkers, + NeedLeaderElection: &LeaderElect, + }). WithEventFilter(predicate.Funcs{ CreateFunc: func(_ event.TypedCreateEvent[k8sclient.Object]) bool { return true }, - UpdateFunc: func(_ event.TypedUpdateEvent[k8sclient.Object]) bool { return true }, + UpdateFunc: func(e event.TypedUpdateEvent[k8sclient.Object]) bool { + oldAnn := e.ObjectOld.GetAnnotations() + newAnn := e.ObjectNew.GetAnnotations() + + if !annotationsEqual(oldAnn, newAnn) { + // Remove metrics for pod, if the annotations have changed + r.Metrics.RemovePod(e.ObjectOld.GetNamespace(), e.ObjectOld.GetName()) + } + return true + }, DeleteFunc: func(e event.TypedDeleteEvent[k8sclient.Object]) bool { r.Log.Infof("Pod deleted: %s/%s", e.Object.GetNamespace(), e.Object.GetName()) r.Metrics.RemovePod(e.Object.GetNamespace(), e.Object.GetName()) @@ -105,3 +118,16 @@ func (r *PodReconciler) SetupWithManager(mgr ctrl.Manager) error { }). Complete(r) } + +// annotationsEqual compares two annotation maps for equality. +func annotationsEqual(a, b map[string]string) bool { + if len(a) != len(b) { + return false + } + for key, valA := range a { + if valB, ok := b[key]; !ok || valA != valB { + return false + } + } + return true +} diff --git a/pkg/controller/pod_controller_test.go b/pkg/controller/pod_controller_test.go index ab33fb3e..f3e45842 100644 --- a/pkg/controller/pod_controller_test.go +++ b/pkg/controller/pod_controller_test.go @@ -41,13 +41,14 @@ func TestNewController(t *testing.T) { ) imageClient := &client.Client{} - controller := NewPodReconciler(5*time.Minute, metrics, imageClient, kubeClient, testLogger, true) + controller := NewPodReconciler(5*time.Minute, metrics, imageClient, kubeClient, testLogger, time.Hour, true) assert.NotNil(t, controller) assert.Equal(t, controller.defaultTestAll, true) assert.Equal(t, controller.Client, kubeClient) assert.NotNil(t, controller.VersionChecker) } + func TestReconcile(t *testing.T) { imageClient := &client.Client{} @@ -84,8 +85,7 @@ func TestReconcile(t *testing.T) { prometheus.NewRegistry(), kubeClient, ) - controller := NewPodReconciler(5*time.Minute, metrics, imageClient, kubeClient, testLogger, true) - controller.RequeueDuration = 5 * time.Minute + controller := NewPodReconciler(5*time.Minute, metrics, imageClient, kubeClient, testLogger, 5*time.Minute, true) ctx := context.Background() @@ -113,6 +113,7 @@ func TestReconcile(t *testing.T) { }) } } + func TestSetupWithManager(t *testing.T) { kubeClient := fake.NewClientBuilder().Build() metrics := metrics.New( @@ -121,7 +122,7 @@ func TestSetupWithManager(t *testing.T) { kubeClient, ) imageClient := &client.Client{} - controller := NewPodReconciler(5*time.Minute, metrics, imageClient, kubeClient, testLogger, true) + controller := NewPodReconciler(5*time.Minute, metrics, imageClient, kubeClient, testLogger, time.Hour, true) mgr, err := manager.New(&rest.Config{}, manager.Options{LeaderElectionConfig: nil}) require.NoError(t, err) diff --git a/pkg/controller/search/search.go b/pkg/controller/search/search.go index e75daccd..79e4b49a 100644 --- a/pkg/controller/search/search.go +++ b/pkg/controller/search/search.go @@ -20,6 +20,9 @@ type Searcher interface { ResolveSHAToTag(ctx context.Context, imageURL string, imageSHA string) (string, error) } +// Ensure The search Struct implements a cacheHandler +var _ cache.Handler = (*Search)(nil) + // Search is the implementation for the searching and caching of image URLs. type Search struct { log *logrus.Entry @@ -40,7 +43,7 @@ func New(log *logrus.Entry, cacheTimeout time.Duration, versionGetter *version.V return s } -func (s *Search) Fetch(ctx context.Context, imageURL string, opts *api.Options) (interface{}, error) { +func (s *Search) Fetch(ctx context.Context, imageURL string, opts *api.Options) (any, error) { latestImage, err := s.versionGetter.LatestTagFromImage(ctx, imageURL, opts) if err != nil { return nil, err @@ -67,7 +70,6 @@ func (s *Search) LatestImage(ctx context.Context, imageURL string, opts *api.Opt } func (s *Search) ResolveSHAToTag(ctx context.Context, imageURL string, imageSHA string) (string, error) { - tag, err := s.versionGetter.ResolveSHAToTag(ctx, imageURL, imageSHA) if err != nil { return "", fmt.Errorf("failed to resolve sha to tag: %w", err) diff --git a/pkg/version/filters.go b/pkg/version/filters.go index f8360581..314c325d 100644 --- a/pkg/version/filters.go +++ b/pkg/version/filters.go @@ -15,6 +15,10 @@ func isSBOMAttestationOrSig(tag string) bool { // Used when filtering Tags as a SemVer func shouldSkipTag(opts *api.Options, v *semver.SemVer) bool { + if isSBOMAttestationOrSig(v.String()) { + return true + } + // Handle Regex matching if opts.RegexMatcher != nil { return !opts.RegexMatcher.MatchString(v.String()) diff --git a/pkg/version/helpers.go b/pkg/version/helpers.go new file mode 100644 index 00000000..d37f8ca5 --- /dev/null +++ b/pkg/version/helpers.go @@ -0,0 +1,55 @@ +package version + +import ( + "fmt" + + "github.com/jetstack/version-checker/pkg/api" + "github.com/jetstack/version-checker/pkg/version/semver" +) + +// latestSemver will return the latest ImageTag based on the given options +// restriction, using semver. This should not be used if UseSHA has been +// enabled. +func latestSemver(opts *api.Options, tags []api.ImageTag) (*api.ImageTag, error) { + var ( + latestImageTag *api.ImageTag + latestV *semver.SemVer + ) + + for i := range tags { + v := semver.Parse(tags[i].Tag) + + if shouldSkipTag(opts, v) { + continue + } + + if isBetterSemVer(opts, latestV, v, latestImageTag, &tags[i]) { + latestV = v + latestImageTag = &tags[i] + } + } + + if latestImageTag == nil { + return nil, fmt.Errorf("no suitable version found") + } + + return latestImageTag, nil +} + +// latestSHA will return the latest ImageTag based on image timestamps. +func latestSHA(opts *api.Options, tags []api.ImageTag) (*api.ImageTag, error) { + var latestTag *api.ImageTag + + for i := range tags { + // Filter out SBOM and Attestation/Sig's... + if shouldSkipSHA(opts, tags[i].Tag) { + continue + } + + if latestTag == nil || tags[i].Timestamp.After(latestTag.Timestamp) { + latestTag = &tags[i] + } + } + + return latestTag, nil +} diff --git a/pkg/version/semver/semver.go b/pkg/version/semver/semver.go index 0fb35d9a..ad74f98e 100644 --- a/pkg/version/semver/semver.go +++ b/pkg/version/semver/semver.go @@ -12,15 +12,15 @@ var ( // SemVer is a struct to contain a SemVer of an image tag. type SemVer struct { - // version is the version number of a tag. 'Left', or smaller index, the - // higher weight. - version [3]int64 // metadata holds the metadata, which is the string suffixed from the patch metadata string // original holds the origin string of the tag original string + // version is the version number of a tag. 'Left', or smaller index, the + // higher weight. + version [3]int64 } func Parse(tag string) *SemVer { diff --git a/pkg/version/version.go b/pkg/version/version.go index b34a846d..b0fb5ebd 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -13,9 +13,10 @@ import ( "github.com/jetstack/version-checker/pkg/cache" versionerrors "github.com/jetstack/version-checker/pkg/version/errors" - "github.com/jetstack/version-checker/pkg/version/semver" ) +var _ cache.Handler = (*Version)(nil) + type Version struct { log *logrus.Entry @@ -55,8 +56,8 @@ func (v *Version) LatestTagFromImage(ctx context.Context, imageURL string, opts } if tag == nil { - return nil, versionerrors.NewVersionErrorNotFound("%s: failed to find latest image based on SHA", - imageURL) + return nil, versionerrors.NewVersionErrorNotFound( + "%s: failed to find latest image based on SHA", imageURL) } } else { tag, err = latestSemver(opts, tags) @@ -66,7 +67,8 @@ func (v *Version) LatestTagFromImage(ctx context.Context, imageURL string, opts if tag == nil { optsBytes, _ := json.Marshal(opts) - return nil, versionerrors.NewVersionErrorNotFound("%s: no tags found with these option constraints: %s", + return nil, versionerrors.NewVersionErrorNotFound( + "%s: no tags found with these option constraints: %s", imageURL, optsBytes) } } @@ -76,7 +78,6 @@ func (v *Version) LatestTagFromImage(ctx context.Context, imageURL string, opts // ResolveSHAToTag Resolve a SHA to a tag if possible func (v *Version) ResolveSHAToTag(ctx context.Context, imageURL string, imageSHA string) (string, error) { - tagsI, err := v.imageCache.Get(ctx, imageURL, imageURL, nil) if err != nil { return "", err @@ -84,7 +85,7 @@ func (v *Version) ResolveSHAToTag(ctx context.Context, imageURL string, imageSHA tags := tagsI.([]api.ImageTag) for i := range tags { - if tags[i].SHA == imageSHA { + if tags[i].MatchesSHA(imageSHA) { return tags[i].Tag, nil } } @@ -110,55 +111,3 @@ func (v *Version) Fetch(ctx context.Context, imageURL string, _ *api.Options) (i return tags, nil } - -// latestSemver will return the latest ImageTag based on the given options -// restriction, using semver. This should not be used is UseSHA has been -// enabled. -func latestSemver(opts *api.Options, tags []api.ImageTag) (*api.ImageTag, error) { - var ( - latestImageTag *api.ImageTag - latestV *semver.SemVer - ) - - for i := range tags { - // Filter out SBOM and Attestation/Sig's - if isSBOMAttestationOrSig(tags[i].Tag) || isSBOMAttestationOrSig(tags[i].SHA) { - continue - } - - v := semver.Parse(tags[i].Tag) - - if shouldSkipTag(opts, v) { - continue - } - - if isBetterSemVer(opts, latestV, v, latestImageTag, &tags[i]) { - latestV = v - latestImageTag = &tags[i] - } - } - - if latestImageTag == nil { - return nil, fmt.Errorf("no suitable version found") - } - - return latestImageTag, nil -} - -// latestSHA will return the latest ImageTag based on image timestamps. -func latestSHA(opts *api.Options, tags []api.ImageTag) (*api.ImageTag, error) { - var latestTag *api.ImageTag - - for i := range tags { - // Filter out SBOM and Attestation/Sig's... - if shouldSkipSHA(opts, tags[i].Tag) { - continue - } - - if latestTag == nil || tags[i].Timestamp.After(latestTag.Timestamp) { - latestTag = &tags[i] - } - } - - return latestTag, nil -} From e6f8436dc9b9aec8e5ea288e37c9be2bf48b6564 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 29 Apr 2025 09:48:58 +0100 Subject: [PATCH 2/7] Apply suggestions from code review --- cmd/app/options.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmd/app/options.go b/cmd/app/options.go index 94dff98e..9d43a6f0 100644 --- a/cmd/app/options.go +++ b/cmd/app/options.go @@ -369,9 +369,6 @@ func (o *Options) assignSelfhosted(envs []string) { } initOptions := func(name string) { - if name == "" { - panic("Not meant to be empty!") - } if o.Client.Selfhosted[name] == nil { o.Client.Selfhosted[name] = new(selfhosted.Options) } From 07571f988fba7cf0f0ff99d9237af667249e2056 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 29 Apr 2025 13:51:22 +0100 Subject: [PATCH 3/7] Fix options parsing - ensuring that TOKEN_PATH and TOKEN_XXX aren't mixed (#367) --- cmd/app/options.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/app/options.go b/cmd/app/options.go index 9d43a6f0..ba3cf43a 100644 --- a/cmd/app/options.go +++ b/cmd/app/options.go @@ -74,9 +74,9 @@ type Options struct { selfhosted selfhosted.Options CacheTimeout time.Duration - RequeueDuration time.Duration GracefulShutdownTimeout time.Duration CacheSyncPeriod time.Duration + RequeueDuration time.Duration DefaultTestAll bool } @@ -446,9 +446,11 @@ func (o *Options) assignSelfhosted(envs []string) { } } + // If we have some selfhosted flags, lets set them here... if len(o.selfhosted.Host) > 0 { o.Client.Selfhosted[o.selfhosted.Host] = &o.selfhosted } + if !validSelfHostedOpts(o) { panic(fmt.Errorf("invalid self hosted configuration")) } From 565dfe76095024e31b6291845ba6d92bb62a4cce Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 29 Apr 2025 13:54:13 +0100 Subject: [PATCH 4/7] Fix issue 362 (#365) --- pkg/metrics/helpers.go | 55 +++++++++++++++++++++++++ pkg/metrics/metrics.go | 81 +++++++++---------------------------- pkg/metrics/metrics_test.go | 38 +++++++++++++++-- 3 files changed, 109 insertions(+), 65 deletions(-) create mode 100644 pkg/metrics/helpers.go diff --git a/pkg/metrics/helpers.go b/pkg/metrics/helpers.go new file mode 100644 index 00000000..ea3e2ff9 --- /dev/null +++ b/pkg/metrics/helpers.go @@ -0,0 +1,55 @@ +package metrics + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/prometheus/client_golang/prometheus" +) + +func buildFullLabels(namespace, pod, container, containerType, imageURL, currentVersion, latestVersion string) prometheus.Labels { + return prometheus.Labels{ + "namespace": namespace, + "pod": pod, + "container_type": containerType, + "container": container, + "image": imageURL, + "current_version": currentVersion, + "latest_version": latestVersion, + } +} + +func buildLastUpdatedLabels(namespace, pod, container, containerType, imageURL string) prometheus.Labels { + return prometheus.Labels{ + "namespace": namespace, + "pod": pod, + "container_type": containerType, + "container": container, + "image": imageURL, + } +} + +func buildPodPartialLabels(namespace, pod string) prometheus.Labels { + return prometheus.Labels{ + "namespace": namespace, + "pod": pod, + } +} + +func buildContainerPartialLabels(namespace, pod, container, containerType string) prometheus.Labels { + return prometheus.Labels{ + "namespace": namespace, + "pod": pod, + "container": container, + "container_type": containerType, + } +} + +// This _should_ leverage the Controllers Cache +func (m *Metrics) PodExists(ctx context.Context, ns, name string) bool { + pod := &corev1.Pod{} + err := m.cache.Get(ctx, types.NamespacedName{Name: name, Namespace: ns}, pod) + return err == nil && pod.GetDeletionTimestamp() == nil +} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 09aa10ee..c9324c14 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -7,8 +7,6 @@ import ( "github.com/sirupsen/logrus" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" k8sclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/prometheus/client_golang/prometheus" @@ -17,6 +15,8 @@ import ( ctrmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" ) +const MetricNamespace = "version_checker" + // Metrics is used to expose container image version checks as prometheus // metrics. type Metrics struct { @@ -39,12 +39,13 @@ type Metrics struct { // func New(log *logrus.Entry, reg ctrmetrics.RegistererGatherer, kubeClient k8sclient.Client) *Metrics { func New(log *logrus.Entry, reg ctrmetrics.RegistererGatherer, cache k8sclient.Reader) *Metrics { // Attempt to register, but ignore errors + // TODO: We should check for AlreadyRegisteredError err type here for better error handling _ = reg.Register(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) _ = reg.Register(collectors.NewGoCollector()) containerImageVersion := promauto.With(reg).NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "version_checker", + Namespace: MetricNamespace, Name: "is_latest_version", Help: "Where the container in use is using the latest upstream registry version", }, @@ -54,7 +55,7 @@ func New(log *logrus.Entry, reg ctrmetrics.RegistererGatherer, cache k8sclient.R ) containerImageChecked := promauto.With(reg).NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "version_checker", + Namespace: MetricNamespace, Name: "last_checked", Help: "Timestamp when the image was checked", }, @@ -64,7 +65,7 @@ func New(log *logrus.Entry, reg ctrmetrics.RegistererGatherer, cache k8sclient.R ) containerImageDuration := promauto.With(reg).NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "version_checker", + Namespace: MetricNamespace, Name: "image_lookup_duration", Help: "Time taken to lookup version.", }, @@ -72,7 +73,7 @@ func New(log *logrus.Entry, reg ctrmetrics.RegistererGatherer, cache k8sclient.R ) containerImageErrors := promauto.With(reg).NewCounterVec( prometheus.CounterOpts{ - Namespace: "version_checker", + Namespace: MetricNamespace, Name: "image_failures_total", Help: "Total number of errors where the version-checker was unable to get the latest upstream registry version", }, @@ -104,12 +105,12 @@ func (m *Metrics) AddImage(namespace, pod, container, containerType, imageURL st } m.containerImageVersion.With( - m.buildFullLabels(namespace, pod, container, containerType, imageURL, currentVersion, latestVersion), + buildFullLabels(namespace, pod, container, containerType, imageURL, currentVersion, latestVersion), ).Set(isLatestF) // Bump last updated timestamp m.containerImageChecked.With( - m.buildLastUpdatedLabels(namespace, pod, container, containerType, imageURL), + buildLastUpdatedLabels(namespace, pod, container, containerType, imageURL), ).Set(float64(time.Now().Unix())) } @@ -118,20 +119,14 @@ func (m *Metrics) RemoveImage(namespace, pod, container, containerType string) { defer m.mu.Unlock() total := 0 - total += m.containerImageVersion.DeletePartialMatch( - m.buildPartialLabels(namespace, pod), - ) - total += m.containerImageDuration.DeletePartialMatch( - m.buildPartialLabels(namespace, pod), - ) + labels := buildContainerPartialLabels(namespace, pod, container, containerType) - total += m.containerImageChecked.DeletePartialMatch( - m.buildPartialLabels(namespace, pod), - ) - total += m.containerImageErrors.DeletePartialMatch( - m.buildPartialLabels(namespace, pod), - ) - m.log.Infof("Removed %d metrics for image %s/%s/%s", total, namespace, pod, container) + total += m.containerImageVersion.DeletePartialMatch(labels) + total += m.containerImageDuration.DeletePartialMatch(labels) + total += m.containerImageChecked.DeletePartialMatch(labels) + total += m.containerImageErrors.DeletePartialMatch(labels) + + m.log.Infof("Removed %d metrics for image %s/%s/%s (%s)", total, namespace, pod, container, containerType) } func (m *Metrics) RemovePod(namespace, pod string) { @@ -140,16 +135,16 @@ func (m *Metrics) RemovePod(namespace, pod string) { total := 0 total += m.containerImageVersion.DeletePartialMatch( - m.buildPartialLabels(namespace, pod), + buildPodPartialLabels(namespace, pod), ) total += m.containerImageDuration.DeletePartialMatch( - m.buildPartialLabels(namespace, pod), + buildPodPartialLabels(namespace, pod), ) total += m.containerImageChecked.DeletePartialMatch( - m.buildPartialLabels(namespace, pod), + buildPodPartialLabels(namespace, pod), ) total += m.containerImageErrors.DeletePartialMatch( - m.buildPartialLabels(namespace, pod), + buildPodPartialLabels(namespace, pod), ) m.log.Infof("Removed %d metrics for pod %s/%s", total, namespace, pod) @@ -182,39 +177,3 @@ func (m *Metrics) ReportError(namespace, pod, container, imageURL string) { namespace, pod, container, imageURL, ).Inc() } - -func (m *Metrics) buildFullLabels(namespace, pod, container, containerType, imageURL, currentVersion, latestVersion string) prometheus.Labels { - return prometheus.Labels{ - "namespace": namespace, - "pod": pod, - "container_type": containerType, - "container": container, - "image": imageURL, - "current_version": currentVersion, - "latest_version": latestVersion, - } -} - -func (m *Metrics) buildLastUpdatedLabels(namespace, pod, container, containerType, imageURL string) prometheus.Labels { - return prometheus.Labels{ - "namespace": namespace, - "pod": pod, - "container_type": containerType, - "container": container, - "image": imageURL, - } -} - -func (m *Metrics) buildPartialLabels(namespace, pod string) prometheus.Labels { - return prometheus.Labels{ - "namespace": namespace, - "pod": pod, - } -} - -// This _should_ leverage the Controllers Cache -func (m *Metrics) PodExists(ctx context.Context, ns, name string) bool { - pod := &corev1.Pod{} - err := m.cache.Get(ctx, types.NamespacedName{Name: name, Namespace: ns}, pod) - return err == nil && pod.GetDeletionTimestamp() == nil -} diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go index 06b5f318..8fe0efd9 100644 --- a/pkg/metrics/metrics_test.go +++ b/pkg/metrics/metrics_test.go @@ -36,7 +36,7 @@ func TestCache(t *testing.T) { for i, typ := range []string{"init", "container"} { version := fmt.Sprintf("0.1.%d", i) mt, _ := m.containerImageVersion.GetMetricWith( - m.buildFullLabels("namespace", "pod", "container", typ, "url", version, version), + buildFullLabels("namespace", "pod", "container", typ, "url", version, version), ) count := testutil.ToFloat64(mt) assert.Equal(t, count, float64(1), "Expected to get a metric for containerImageVersion") @@ -44,7 +44,7 @@ func TestCache(t *testing.T) { // as well as the lastUpdated... for _, typ := range []string{"init", "container"} { - mt, err := m.containerImageChecked.GetMetricWith(m.buildLastUpdatedLabels("namespace", "pod", "container", typ, "url")) + mt, err := m.containerImageChecked.GetMetricWith(buildLastUpdatedLabels("namespace", "pod", "container", typ, "url")) require.NoError(t, err) count := testutil.ToFloat64(mt) assert.GreaterOrEqual(t, count, float64(time.Now().Unix())) @@ -58,14 +58,14 @@ func TestCache(t *testing.T) { for i, typ := range []string{"init", "container"} { version := fmt.Sprintf("0.1.%d", i) mt, _ := m.containerImageVersion.GetMetricWith( - m.buildFullLabels("namespace", "pod", "container", typ, "url", version, version), + buildFullLabels("namespace", "pod", "container", typ, "url", version, version), ) count := testutil.ToFloat64(mt) assert.Equal(t, count, float64(0), "Expected NOT to get a metric for containerImageVersion") } // And the Last Updated is removed too for _, typ := range []string{"init", "container"} { - mt, err := m.containerImageChecked.GetMetricWith(m.buildLastUpdatedLabels("namespace", "pod", "container", typ, "url")) + mt, err := m.containerImageChecked.GetMetricWith(buildLastUpdatedLabels("namespace", "pod", "container", typ, "url")) require.NoError(t, err) count := testutil.ToFloat64(mt) assert.Equal(t, count, float64(0), "Expected to get a metric for containerImageChecked") @@ -184,3 +184,33 @@ func Test_Metrics_SkipOnDeletedPod(t *testing.T) { assert.NotContains(t, *mf.Name, "image_failures_total", "Should not have been found: %+v", mf) } } + +func TestPodAnnotationsChangeAfterRegistration(t *testing.T) { + // Step 2: Create Metrics with fake registry + reg := prometheus.NewRegistry() + log := logrus.NewEntry(logrus.New()) + client := fake.NewClientBuilder().Build() + metrics := New(log, reg, client) + + // Register Metrics... + metrics.AddImage("default", "mypod", "my-init-container", "init", "alpine:latest", false, "1.0", "1.1") + metrics.AddImage("default", "mypod", "mycontainer", "container", "nginx:1.0", true, "1.0", "1.0") + metrics.AddImage("default", "mypod", "sidecar", "container", "alpine:1.0", false, "1.0", "1.1") + + _, err := reg.Gather() + require.NoError(t, err, "Failed to gather metrics") + + assert.Equal(t, 3, + testutil.CollectAndCount(metrics.containerImageVersion.MetricVec, MetricNamespace+"_is_latest_version"), + ) + + // Pod Annotations are changed, only the `mycontainer` should be checked... + + // Remove Init and sidecar + metrics.RemoveImage("default", "mypod", "my-init-container", "init") + metrics.RemoveImage("default", "mypod", "sidecar", "container") + + assert.Equal(t, 1, + testutil.CollectAndCount(metrics.containerImageVersion.MetricVec, MetricNamespace+"_is_latest_version"), + ) +} From 56ee08a81a5d3b5dc74ebe8770af5229913e186f Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 29 Apr 2025 16:06:07 +0100 Subject: [PATCH 5/7] Re-sorting arguments for better readability --- cmd/app/options.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/app/options.go b/cmd/app/options.go index f8e9ec26..f358ac25 100644 --- a/cmd/app/options.go +++ b/cmd/app/options.go @@ -126,6 +126,10 @@ func (o *Options) addAppFlags(fs *pflag.FlagSet) { "If enabled, all containers will be tested, unless they have the "+ fmt.Sprintf(`annotation "%s/${my-container}=false".`, api.EnableAnnotationKey)) + fs.StringVarP(&o.LogLevel, + "log-level", "v", "info", + "Log level (debug, info, warn, error, fatal, panic).") + fs.DurationVarP(&o.CacheTimeout, "image-cache-timeout", "c", time.Minute*30, "The time for an image version in the cache to be considered fresh. Images "+ @@ -135,17 +139,13 @@ func (o *Options) addAppFlags(fs *pflag.FlagSet) { "requeue-duration", "r", time.Hour, "The time a pod will be re-checked for new versions/tags") - fs.StringVarP(&o.LogLevel, - "log-level", "v", "info", - "Log level (debug, info, warn, error, fatal, panic).") + fs.DurationVarP(&o.CacheSyncPeriod, + "cache-sync-period", "", 5*time.Hour, + "The time in which all resources should be updated.") fs.DurationVarP(&o.GracefulShutdownTimeout, "graceful-shutdown-timeout", "", 10*time.Second, "Time that the manager should wait for all controller to shutdown.") - - fs.DurationVarP(&o.CacheSyncPeriod, - "cache-sync-period", "", 5*time.Hour, - "The time in which all resources should be updated.") } func (o *Options) addAuthFlags(fs *pflag.FlagSet) { From 38384682203afbb3d2400b64ac2ebc5b7e7a52af Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 1 May 2025 16:47:52 +0100 Subject: [PATCH 6/7] Rename to BuildTags --- pkg/client/acr/acr.go | 2 +- pkg/client/ecr/ecr.go | 2 +- pkg/client/selfhosted/selfhosted.go | 6 +++--- pkg/client/util/util.go | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/client/acr/acr.go b/pkg/client/acr/acr.go index cf91a88b..03701f43 100644 --- a/pkg/client/acr/acr.go +++ b/pkg/client/acr/acr.go @@ -94,7 +94,7 @@ func (c *Client) Tags(ctx context.Context, host, repo, image string) ([]api.Imag current := base // copy the base current.Tag = tag // set tag value - util.FindParentTags(tags, tag, ¤t) + util.BuildTags(tags, tag, ¤t) } } return util.TagMaptoList(tags), nil diff --git a/pkg/client/ecr/ecr.go b/pkg/client/ecr/ecr.go index 431065d1..49367d50 100644 --- a/pkg/client/ecr/ecr.go +++ b/pkg/client/ecr/ecr.go @@ -84,7 +84,7 @@ func (c *Client) Tags(ctx context.Context, host, repo, image string) ([]api.Imag current := base // copy the base current.Tag = tag // set tag value - util.FindParentTags(tags, tag, ¤t) + util.BuildTags(tags, tag, ¤t) } } diff --git a/pkg/client/selfhosted/selfhosted.go b/pkg/client/selfhosted/selfhosted.go index abfb2b40..ef475e16 100644 --- a/pkg/client/selfhosted/selfhosted.go +++ b/pkg/client/selfhosted/selfhosted.go @@ -218,17 +218,17 @@ func (c *Client) Tags(ctx context.Context, host, repo, image string) ([]api.Imag Architecture: api.Architecture(manifestResponse.Architecture), } - util.FindParentTags(tags, tag, ¤t) + util.BuildTags(tags, tag, ¤t) for _, manifest := range manifestListResponse.Manifests { // If we didn't get a SHA from the inital call, // lets set it from the manifestList - if current.SHA != "" && manifest.Digest != "" { + if current.SHA == "" && manifest.Digest != "" { current.SHA = manifest.Digest } - util.FindParentTags(tags, tag, ¤t) + util.BuildTags(tags, tag, ¤t) } } return util.TagMaptoList(tags), nil diff --git a/pkg/client/util/util.go b/pkg/client/util/util.go index b29e2f98..cdce903e 100644 --- a/pkg/client/util/util.go +++ b/pkg/client/util/util.go @@ -81,7 +81,7 @@ func TagMaptoList(tags map[string]api.ImageTag) []api.ImageTag { return taglist } -func FindParentTags(tags map[string]api.ImageTag, tag string, current *api.ImageTag) { +func BuildTags(tags map[string]api.ImageTag, tag string, current *api.ImageTag) { // Already exists — add as child if parent, exists := tags[tag]; exists { parent.Children = append(parent.Children, current) From 108f1aa7b51f179c3003875a2d9f6ad9268581fb Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 1 May 2025 16:49:24 +0100 Subject: [PATCH 7/7] Remove old/commented test --- pkg/client/selfhosted/selfhosted_test.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/pkg/client/selfhosted/selfhosted_test.go b/pkg/client/selfhosted/selfhosted_test.go index 3aa30bb4..d9144130 100644 --- a/pkg/client/selfhosted/selfhosted_test.go +++ b/pkg/client/selfhosted/selfhosted_test.go @@ -59,17 +59,6 @@ func TestNew(t *testing.T) { assert.Contains(t, err.Error(), "failed parsing url") }) - // t.Run("Error on missing host", func(t *testing.T) { - // opts := &Options{ - // Host: "", - // CAPath: "invalid/path", - // } - // client, err := New(ctx, log, opts) - // assert.Nil(t, client) - // assert.Error(t, err) - // assert.Contains(t, err.Error(), "host cannot be empty") - // }) - t.Run("error on username/password and bearer token both specified", func(t *testing.T) { opts := &Options{ Host: "https://testregistry.com",