diff --git a/cmd/app/options.go b/cmd/app/options.go index df27c425..f358ac25 100644 --- a/cmd/app/options.go +++ b/cmd/app/options.go @@ -126,18 +126,14 @@ func (o *Options) addAppFlags(fs *pflag.FlagSet) { "If enabled, all containers will be tested, unless they have the "+ fmt.Sprintf(`annotation "%s/${my-container}=false".`, api.EnableAnnotationKey)) - fs.DurationVarP(&o.CacheTimeout, - "image-cache-timeout", "c", time.Minute*30, - "The time for an image version in the cache to be considered fresh. Images "+ - "will be rechecked after this interval.") - fs.StringVarP(&o.LogLevel, "log-level", "v", "info", "Log level (debug, info, warn, error, fatal, panic).") - fs.DurationVarP(&o.GracefulShutdownTimeout, - "graceful-shutdown-timeout", "", 10*time.Second, - "Time that the manager should wait for all controller to shutdown.") + fs.DurationVarP(&o.CacheTimeout, + "image-cache-timeout", "c", time.Minute*30, + "The time for an image version in the cache to be considered fresh. Images "+ + "will be rechecked after this interval.") fs.DurationVarP(&o.RequeueDuration, "requeue-duration", "r", time.Hour, @@ -146,6 +142,10 @@ func (o *Options) addAppFlags(fs *pflag.FlagSet) { fs.DurationVarP(&o.CacheSyncPeriod, "cache-sync-period", "", 5*time.Hour, "The time in which all resources should be updated.") + + fs.DurationVarP(&o.GracefulShutdownTimeout, + "graceful-shutdown-timeout", "", 10*time.Second, + "Time that the manager should wait for all controller to shutdown.") } func (o *Options) addAuthFlags(fs *pflag.FlagSet) { diff --git a/go.mod b/go.mod index 149bc99b..65d60607 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,6 @@ require ( github.com/bombsimon/logrusr/v4 v4.1.0 github.com/go-chi/transport v0.5.0 github.com/gofri/go-github-ratelimit v1.1.1 - github.com/google/go-cmp v0.7.0 github.com/google/go-containerregistry v0.20.3 github.com/google/go-containerregistry/pkg/authn/k8schain v0.0.0-20250225234217-098045d5e61f github.com/google/go-github/v70 v70.0.0 @@ -43,6 +42,7 @@ require ( github.com/jarcoal/httpmock v1.4.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/stretchr/testify v1.10.0 + golang.org/x/time v0.11.0 sigs.k8s.io/controller-runtime v0.20.4 ) @@ -92,6 +92,7 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/google/btree v1.1.3 // indirect github.com/google/gnostic-models v0.6.9 // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/google/go-containerregistry/pkg/authn/kubernetes v0.0.0-20241111191718-6bce25ecf029 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect @@ -128,7 +129,6 @@ require ( golang.org/x/sys v0.31.0 // indirect golang.org/x/term v0.30.0 // indirect golang.org/x/text v0.23.0 // indirect - golang.org/x/time v0.11.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect google.golang.org/protobuf v1.36.6 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect diff --git a/pkg/api/options.go b/pkg/api/options.go index 8590aff1..6b66d162 100644 --- a/pkg/api/options.go +++ b/pkg/api/options.go @@ -7,20 +7,20 @@ import "regexp" type Options struct { OverrideURL *string `json:"override-url,omitempty"` + MatchRegex *string `json:"match-regex,omitempty"` + + PinMajor *int64 `json:"pin-major,omitempty"` + PinMinor *int64 `json:"pin-minor,omitempty"` + PinPatch *int64 `json:"pin-patch,omitempty"` + + RegexMatcher *regexp.Regexp `json:"-"` + // UseSHA cannot be used with any other options UseSHA bool `json:"use-sha,omitempty"` // Resolve SHA to a TAG ResolveSHAToTags bool `json:"resolve-sha-to-tags,omitempty"` - MatchRegex *string `json:"match-regex,omitempty"` - // UseMetaData defines whether tags with '-alpha', '-debian.0' etc. is // permissible. UseMetaData bool `json:"use-metadata,omitempty"` - - PinMajor *int64 `json:"pin-major,omitempty"` - PinMinor *int64 `json:"pin-minor,omitempty"` - PinPatch *int64 `json:"pin-patch,omitempty"` - - RegexMatcher *regexp.Regexp `json:"-"` } diff --git a/pkg/api/types.go b/pkg/api/types.go index 903745a1..f4e04875 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -12,6 +12,26 @@ type ImageTag struct { Timestamp time.Time `json:"timestamp"` OS OS `json:"os,omitempty"` Architecture Architecture `json:"architecture,omitempty"` + + // If this is a Manifest list we need to keep them together + Children []*ImageTag `json:"children,omitempty"` +} + +func (i *ImageTag) MatchesSHA(sha string) bool { + if sha == i.SHA { + return true + } + for _, known := range i.Children { + if known.SHA == sha { + return true + } + } + return false +} + +type Platform struct { + OS OS + Architecture Architecture } type OS string diff --git a/pkg/client/acr/acr.go b/pkg/client/acr/acr.go index fa21e4da..03701f43 100644 --- a/pkg/client/acr/acr.go +++ b/pkg/client/acr/acr.go @@ -8,33 +8,26 @@ import ( "io" "net/http" "sync" - "time" - - "github.com/MicahParks/keyfunc/v3" "github.com/Azure/go-autorest/autorest" - "github.com/Azure/go-autorest/autorest/adal" - "github.com/golang-jwt/jwt/v5" "github.com/jetstack/version-checker/pkg/api" "github.com/jetstack/version-checker/pkg/client/util" ) +// Ensure that we are an ImageClient +var _ api.ImageClient = (*Client)(nil) + const ( userAgent = "jetstack/version-checker" requiredScope = "repository:*:metadata_read" ) type Client struct { - Options - - cacheMu sync.Mutex cachedACRClient map[string]*acrClient -} + Options -type acrClient struct { - tokenExpiry time.Time - *autorest.Client + cacheMu sync.Mutex } type Options struct { @@ -44,18 +37,6 @@ type Options struct { JWKSURI string } -type AccessTokenResponse struct { - AccessToken string `json:"access_token"` -} - -type ManifestResponse struct { - Manifests []struct { - Digest string `json:"digest"` - CreatedTime time.Time `json:"createdTime"` - Tags []string `json:"tags"` - } `json:"manifests"` -} - func New(opts Options) (*Client, error) { if len(opts.RefreshToken) > 0 && (len(opts.Username) > 0 || len(opts.Password) > 0) { @@ -90,27 +71,33 @@ func (c *Client) Tags(ctx context.Context, host, repo, image string) ([]api.Imag host, err) } - var tags []api.ImageTag + // Create a map of tags, so that when we come up with additional Tags + // we can add them as Children + tags := map[string]api.ImageTag{} + for _, manifest := range manifestResp.Manifests { - if len(manifest.Tags) == 0 { - tags = append(tags, api.ImageTag{ - SHA: manifest.Digest, - Timestamp: manifest.CreatedTime, - }) + // Base data shared across tags + base := api.ImageTag{ + SHA: manifest.Digest, + Timestamp: manifest.CreatedTime, + OS: manifest.OS, + Architecture: manifest.Architecture, + } + // No tags, use digest as the key + if len(manifest.Tags) == 0 { + tags[base.SHA] = base continue } for _, tag := range manifest.Tags { - tags = append(tags, api.ImageTag{ - SHA: manifest.Digest, - Timestamp: manifest.CreatedTime, - Tag: tag, - }) + current := base // copy the base + current.Tag = tag // set tag value + + util.BuildTags(tags, tag, ¤t) } } - - return tags, nil + return util.TagMaptoList(tags), nil } func (c *Client) getManifestsWithClient(ctx context.Context, client *acrClient, host, repo, image string) (*http.Response, error) { @@ -148,147 +135,3 @@ func (c *Client) getManifestsWithClient(ctx context.Context, client *acrClient, return resp, nil } - -func (c *Client) getACRClient(ctx context.Context, host string) (*acrClient, error) { - c.cacheMu.Lock() - defer c.cacheMu.Unlock() - - if client, ok := c.cachedACRClient[host]; ok && time.Now().After(client.tokenExpiry) { - return client, nil - } - - var ( - client *acrClient - accessTokenClient *autorest.Client - accessTokenReq *http.Request - err error - ) - if len(c.RefreshToken) > 0 { - accessTokenClient, accessTokenReq, err = c.getAccessTokenRequesterForRefreshToken(ctx, host) - } else { - accessTokenClient, accessTokenReq, err = c.getAccessTokenRequesterForBasicAuth(ctx, host) - } - if err != nil { - return nil, err - } - if client, err = c.getAuthorizedClient(accessTokenClient, accessTokenReq, host); err != nil { - return nil, err - } - - c.cachedACRClient[host] = client - - return client, nil -} - -func (c *Client) getAccessTokenRequesterForBasicAuth(ctx context.Context, host string) (*autorest.Client, *http.Request, error) { - client := autorest.NewClientWithUserAgent(userAgent) - client.Authorizer = autorest.NewBasicAuthorizer(c.Username, c.Password) - urlParameters := map[string]interface{}{ - "url": "https://" + host, - } - - preparer := autorest.CreatePreparer( - autorest.WithCustomBaseURL("{url}", urlParameters), - autorest.WithPath("/oauth2/token"), - autorest.WithQueryParameters(map[string]interface{}{ - "scope": requiredScope, - "service": host, - }), - ) - req, err := preparer.Prepare((&http.Request{}).WithContext(ctx)) - if err != nil { - return nil, nil, err - } - - return &client, req, nil -} - -func (c *Client) getAccessTokenRequesterForRefreshToken(ctx context.Context, host string) (*autorest.Client, *http.Request, error) { - client := autorest.NewClientWithUserAgent(userAgent) - urlParameters := map[string]interface{}{ - "url": "https://" + host, - } - - formDataParameters := map[string]interface{}{ - "grant_type": "refresh_token", - "refresh_token": c.RefreshToken, - "scope": requiredScope, - "service": host, - } - - preparer := autorest.CreatePreparer( - autorest.AsPost(), - autorest.WithCustomBaseURL("{url}", urlParameters), - autorest.WithPath("/oauth2/token"), - autorest.WithFormData(autorest.MapToValues(formDataParameters))) - req, err := preparer.Prepare((&http.Request{}).WithContext(ctx)) - if err != nil { - return nil, nil, err - } - return &client, req, nil -} - -func (c *Client) getAuthorizedClient(client *autorest.Client, req *http.Request, host string) (*acrClient, error) { - resp, err := autorest.SendWithSender(client, req, - autorest.DoRetryForStatusCodes(client.RetryAttempts, client.RetryDuration, autorest.StatusCodesForRetry...), - ) - if err != nil { - return nil, fmt.Errorf("%s: failed to request access token: %s", - host, err) - } - defer func() { _ = resp.Body.Close() }() - - var respToken AccessTokenResponse - if err := json.NewDecoder(resp.Body).Decode(&respToken); err != nil { - return nil, fmt.Errorf("%s: failed to decode access token response: %s", - host, err) - } - - exp, err := c.getTokenExpiration(respToken.AccessToken) - if err != nil { - return nil, fmt.Errorf("%s: %s", host, err) - } - - token := &adal.Token{ - RefreshToken: "", // empty if access_token was retrieved with basic auth. but client is not reused after expiry anyway (see cachedACRClient) - AccessToken: respToken.AccessToken, - } - - client.Authorizer = autorest.NewBearerAuthorizer(token) - - return &acrClient{ - tokenExpiry: exp, - Client: client, - }, nil -} - -func (c *Client) getTokenExpiration(tokenString string) (time.Time, error) { - jwtParser := jwt.NewParser(jwt.WithoutClaimsValidation()) - var token *jwt.Token - var err error - if c.JWKSURI != "" { - var k keyfunc.Keyfunc - k, err = keyfunc.NewDefaultCtx(context.TODO(), []string{c.JWKSURI}) - if err != nil { - return time.Time{}, err - } - token, err = jwtParser.Parse(tokenString, k.Keyfunc) - } else { - token, _, err = jwtParser.ParseUnverified(tokenString, jwt.MapClaims{}) - } - if err != nil { - return time.Time{}, err - } - - claims, ok := token.Claims.(jwt.MapClaims) - if !ok { - return time.Time{}, fmt.Errorf("failed to process claims in access token") - } - - if exp, ok := claims["exp"].(float64); ok { - timestamp := time.Unix(int64(exp), 0) - return timestamp, nil - } - - return time.Time{}, fmt.Errorf("failed to find 'exp' claim in access token") -} diff --git a/pkg/client/acr/auth.go b/pkg/client/acr/auth.go new file mode 100644 index 00000000..5461c3fd --- /dev/null +++ b/pkg/client/acr/auth.go @@ -0,0 +1,163 @@ +package acr + +// The intention here is to provide a client for Azure Container Registry (ACR) +// that can authenticate using either basic authentication (username/password) +// or a refresh token. The client will cache the access token and its expiration +// time to avoid unnecessary requests to the ACR server. + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/Azure/go-autorest/autorest" + "github.com/Azure/go-autorest/autorest/adal" + "github.com/MicahParks/keyfunc/v3" + "github.com/golang-jwt/jwt/v5" +) + +func (c *Client) getACRClient(ctx context.Context, host string) (*acrClient, error) { + c.cacheMu.Lock() + defer c.cacheMu.Unlock() + + if client, ok := c.cachedACRClient[host]; ok && time.Now().After(client.tokenExpiry) { + return client, nil + } + + var ( + client *acrClient + accessTokenClient *autorest.Client + accessTokenReq *http.Request + err error + ) + if len(c.RefreshToken) > 0 { + accessTokenClient, accessTokenReq, err = c.getAccessTokenRequesterForRefreshToken(ctx, host) + } else { + accessTokenClient, accessTokenReq, err = c.getAccessTokenRequesterForBasicAuth(ctx, host) + } + if err != nil { + return nil, err + } + if client, err = c.getAuthorizedClient(accessTokenClient, accessTokenReq, host); err != nil { + return nil, err + } + + c.cachedACRClient[host] = client + + return client, nil +} + +func (c *Client) getAccessTokenRequesterForBasicAuth(ctx context.Context, host string) (*autorest.Client, *http.Request, error) { + client := autorest.NewClientWithUserAgent(userAgent) + client.Authorizer = autorest.NewBasicAuthorizer(c.Username, c.Password) + urlParameters := map[string]interface{}{ + "url": "https://" + host, + } + + preparer := autorest.CreatePreparer( + autorest.WithCustomBaseURL("{url}", urlParameters), + autorest.WithPath("/oauth2/token"), + autorest.WithQueryParameters(map[string]interface{}{ + "scope": requiredScope, + "service": host, + }), + ) + req, err := preparer.Prepare((&http.Request{}).WithContext(ctx)) + if err != nil { + return nil, nil, err + } + + return &client, req, nil +} + +func (c *Client) getAccessTokenRequesterForRefreshToken(ctx context.Context, host string) (*autorest.Client, *http.Request, error) { + client := autorest.NewClientWithUserAgent(userAgent) + urlParameters := map[string]interface{}{ + "url": "https://" + host, + } + + formDataParameters := map[string]interface{}{ + "grant_type": "refresh_token", + "refresh_token": c.RefreshToken, + "scope": requiredScope, + "service": host, + } + + preparer := autorest.CreatePreparer( + autorest.AsPost(), + autorest.WithCustomBaseURL("{url}", urlParameters), + autorest.WithPath("/oauth2/token"), + autorest.WithFormData(autorest.MapToValues(formDataParameters))) + req, err := preparer.Prepare((&http.Request{}).WithContext(ctx)) + if err != nil { + return nil, nil, err + } + return &client, req, nil +} + +func (c *Client) getAuthorizedClient(client *autorest.Client, req *http.Request, host string) (*acrClient, error) { + resp, err := autorest.SendWithSender(client, req, + autorest.DoRetryForStatusCodes(client.RetryAttempts, client.RetryDuration, autorest.StatusCodesForRetry...), + ) + if err != nil { + return nil, fmt.Errorf("%s: failed to request access token: %s", + host, err) + } + defer func() { _ = resp.Body.Close() }() + + var respToken AccessTokenResponse + if err := json.NewDecoder(resp.Body).Decode(&respToken); err != nil { + return nil, fmt.Errorf("%s: failed to decode access token response: %s", + host, err) + } + + exp, err := c.getTokenExpiration(respToken.AccessToken) + if err != nil { + return nil, fmt.Errorf("%s: %s", host, err) + } + + token := &adal.Token{ + RefreshToken: "", // empty if access_token was retrieved with basic auth. but client is not reused after expiry anyway (see cachedACRClient) + AccessToken: respToken.AccessToken, + } + + client.Authorizer = autorest.NewBearerAuthorizer(token) + + return &acrClient{ + tokenExpiry: exp, + Client: client, + }, nil +} + +func (c *Client) getTokenExpiration(tokenString string) (time.Time, error) { + jwtParser := jwt.NewParser(jwt.WithoutClaimsValidation()) + var token *jwt.Token + var err error + if c.JWKSURI != "" { + var k keyfunc.Keyfunc + k, err = keyfunc.NewDefaultCtx(context.TODO(), []string{c.JWKSURI}) + if err != nil { + return time.Time{}, err + } + token, err = jwtParser.Parse(tokenString, k.Keyfunc) + } else { + token, _, err = jwtParser.ParseUnverified(tokenString, jwt.MapClaims{}) + } + if err != nil { + return time.Time{}, err + } + + claims, ok := token.Claims.(jwt.MapClaims) + if !ok { + return time.Time{}, fmt.Errorf("failed to process claims in access token") + } + + if exp, ok := claims["exp"].(float64); ok { + timestamp := time.Unix(int64(exp), 0) + return timestamp, nil + } + + return time.Time{}, fmt.Errorf("failed to find 'exp' claim in access token") +} diff --git a/pkg/client/acr/types.go b/pkg/client/acr/types.go new file mode 100644 index 00000000..bd09cbb5 --- /dev/null +++ b/pkg/client/acr/types.go @@ -0,0 +1,30 @@ +package acr + +import ( + "time" + + "github.com/Azure/go-autorest/autorest" + "github.com/jetstack/version-checker/pkg/api" +) + +type acrClient struct { + tokenExpiry time.Time + *autorest.Client +} + +type AccessTokenResponse struct { + AccessToken string `json:"access_token"` +} + +// API Taken from documentation @ +// https://learn.microsoft.com/en-us/rest/api/containerregistry/manifests/get-list?view=rest-containerregistry-2019-08-15&tabs=HTTP + +type ManifestResponse struct { + Manifests []struct { + CreatedTime time.Time `json:"createdTime"` + Digest string `json:"digest"` + Architecture api.Architecture `json:"architecture,omitempty"` + OS api.OS `json:"os,omitempty"` + Tags []string `json:"tags"` + } `json:"manifests"` +} diff --git a/pkg/client/client.go b/pkg/client/client.go index fd54cd08..219802d6 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -28,21 +28,21 @@ type ClientHandler interface { // Client is a container image registry client to list tags of given image // URLs. type Client struct { - clients []api.ImageClient fallbackClient api.ImageClient - log *logrus.Entry + log *logrus.Entry + clients []api.ImageClient } // Options used to configure client authentication. type Options struct { ACR acr.Options + Docker docker.Options ECR ecr.Options GCR gcr.Options GHCR ghcr.Options - Docker docker.Options - Quay quay.Options OCI oci.Options + Quay quay.Options Selfhosted map[string]*selfhosted.Options Transport http.RoundTripper @@ -79,7 +79,7 @@ func New(ctx context.Context, log *logrus.Entry, opts Options) (*Client, error) } // Create some of the fallback clients - ociclient, err := oci.New(&opts.OCI) + ociclient, err := oci.New(&opts.OCI, log) if err != nil { return nil, fmt.Errorf("failed to create OCI client: %w", err) } diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 7e32945f..61a6081b 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -2,10 +2,10 @@ package client import ( "context" - "reflect" "testing" "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" "github.com/jetstack/version-checker/pkg/api" "github.com/jetstack/version-checker/pkg/client/acr" @@ -181,20 +181,10 @@ func TestFromImageURL(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { client, host, path := handler.fromImageURL(test.url) - if reflect.TypeOf(client) != reflect.TypeOf(test.expClient) { - t.Errorf("unexpected client, exp=%v got=%v", - reflect.TypeOf(test.expClient), reflect.TypeOf(client)) - } - if host != test.expHost { - t.Errorf("unexpected host, exp=%v got=%v", - test.expHost, host) - } - - if path != test.expPath { - t.Errorf("unexpected path, exp=%s got=%s", - test.expPath, path) - } + assert.IsType(t, test.expClient, client) + assert.Equal(t, test.expHost, host) + assert.Equal(t, test.expPath, path) }) } } diff --git a/pkg/client/docker/docker.go b/pkg/client/docker/docker.go index e3951bdd..2302dec6 100644 --- a/pkg/client/docker/docker.go +++ b/pkg/client/docker/docker.go @@ -10,6 +10,8 @@ import ( "strings" "time" + "golang.org/x/time/rate" + "github.com/sirupsen/logrus" retryablehttp "github.com/hashicorp/go-retryablehttp" @@ -17,29 +19,50 @@ import ( "github.com/jetstack/version-checker/pkg/client/util" ) +// Ensure that we are an ImageClient +var _ api.ImageClient = (*Client)(nil) + +// Values taken from: https://docs.docker.com/docker-hub/usage/#abuse-rate-limit +const ( + windowDuration = time.Minute + APIRateLimit = 500 + maxWait = time.Hour +) + const ( loginURL = "https://hub.docker.com/v2/users/login/" lookupURL = "https://registry.hub.docker.com/v2/repositories/%s/%s/tags?page_size=100" ) type Options struct { + Transporter http.RoundTripper Username string Password string Token string - Transporter http.RoundTripper } type Client struct { *http.Client Options + + log *logrus.Entry + limiter *rate.Limiter } func New(opts Options, log *logrus.Entry) (*Client, error) { ctx := context.Background() + + limiter := rate.NewLimiter( + rate.Every(windowDuration/APIRateLimit), + 1, + ) + log = log.WithField("client", "docker") + retryclient := retryablehttp.NewClient() if opts.Transporter != nil { retryclient.HTTPClient.Transport = opts.Transporter } + retryclient.Backoff = util.RateLimitedBackoffLimiter(log, limiter, maxWait) retryclient.HTTPClient.Timeout = 10 * time.Second retryclient.RetryMax = 10 retryclient.RetryWaitMax = 10 * time.Minute @@ -65,6 +88,8 @@ func New(opts Options, log *logrus.Entry) (*Client, error) { return &Client{ Options: opts, Client: client, + log: log, + limiter: limiter, }, nil } @@ -96,13 +121,23 @@ func (c *Client) Tags(ctx context.Context, _, repo, image string) ([]api.ImageTa } } + tag := api.ImageTag{ + Tag: result.Name, + Timestamp: timestamp, + } + + // If we have a Digest, lets set it.. + if result.Digest != "" { + tag.SHA = result.Digest + } + for _, image := range result.Images { // Image without digest contains no real image. if len(image.Digest) == 0 { continue } - tags = append(tags, api.ImageTag{ + tag.Children = append(tag.Children, &api.ImageTag{ Tag: result.Name, SHA: image.Digest, Timestamp: timestamp, @@ -110,6 +145,14 @@ func (c *Client) Tags(ctx context.Context, _, repo, image string) ([]api.ImageTa Architecture: image.Architecture, }) } + + // If we only have one child, and it has a SHA, then lets use that in the parent + if tag.SHA == "" && len(tag.Children) == 1 && tag.Children[0].SHA != "" { + tag.SHA = tag.Children[0].SHA + } + + // Append our Tag at the end... + tags = append(tags, tag) } url = response.Next @@ -129,6 +172,7 @@ func (c *Client) doRequest(ctx context.Context, url string) (*TagResponse, error if len(c.Token) > 0 { req.Header.Add("Authorization", "Bearer "+c.Token) } + req.Header.Set("User-Agent", "version-checker/docker") resp, err := c.Do(req) if err != nil { @@ -161,6 +205,7 @@ func basicAuthSetup(ctx context.Context, client *http.Client, opts Options) (str return "", err } + req.Header.Set("User-Agent", "version-checker/docker") req.Header.Set("Content-Type", "application/json") req = req.WithContext(ctx) diff --git a/pkg/client/docker/docker_test.go b/pkg/client/docker/docker_test.go new file mode 100644 index 00000000..5b3b7fcd --- /dev/null +++ b/pkg/client/docker/docker_test.go @@ -0,0 +1,138 @@ +package docker + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/jetstack/version-checker/pkg/api" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/sirupsen/logrus" +) + +type hostnameOverride struct { + Host string + RT http.RoundTripper +} + +func (r *hostnameOverride) RoundTrip(req *http.Request) (*http.Response, error) { + if req.Host != r.Host { + if testing.Verbose() { + fmt.Printf("Overriding URI from: %s to %s\n", req.Host, strings.TrimPrefix(r.Host, "http://")) + } + req.Host = strings.TrimPrefix(r.Host, "http://") + req.URL.Host = strings.TrimPrefix(r.Host, "http://") + req.URL.Scheme = "http" + } + // fmt.Printf("Req: %+v", req) + return r.RT.RoundTrip(req) +} + +func TestTags(t *testing.T) { + log := logrus.NewEntry(logrus.New()) + ctx := context.Background() + + t.Run("successful Tags fetch", func(t *testing.T) { + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.NotEmpty(t, r.Header.Get("Authorization")) + + require.Equal(t, "/v2/repositories/testrepo/testimage/tags", r.URL.Path) + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(TagResponse{ + Results: []Result{ + { + Name: "v1.0.0", + Timestamp: time.Now().Add(-24 * time.Hour).Format(time.RFC3339Nano), + Digest: "sha256:abcdef", + Images: []Image{ + {Digest: "sha256:child1", OS: "linux", Architecture: "amd64"}, + }, + }, + { + Name: "v2.0.0", + Timestamp: time.Now().Add(-48 * time.Hour).Format(time.RFC3339Nano), + Images: []Image{ + {Digest: "sha256:child2", OS: "linux", Architecture: "amd64"}, + }, + }, + }, + }) + })) + defer server.Close() + + client := &Client{ + Client: server.Client(), + log: log, + Options: Options{ + Token: "testtoken", + }, + } + + client.Transport = &hostnameOverride{RT: server.Client().Transport, Host: server.URL} + + tags, err := client.Tags(ctx, "NOT USED!", "testrepo", "testimage") + require.NoError(t, err) + require.Len(t, tags, 2) + + assert.Equal(t, "v1.0.0", tags[0].Tag) + assert.Equal(t, "sha256:abcdef", tags[0].SHA) + assert.Equal(t, api.OS("linux"), tags[0].Children[0].OS) + assert.Equal(t, api.Architecture("amd64"), tags[0].Children[0].Architecture) + + assert.Equal(t, "v2.0.0", tags[1].Tag) + assert.NotEmpty(t, tags[1].SHA) + }) + + t.Run("error on invalid response", func(t *testing.T) { + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("invalid json")) + })) + defer server.Close() + + client := &Client{ + Client: server.Client(), + log: log, + Options: Options{ + Token: "testtoken", + }, + } + client.Transport = &hostnameOverride{RT: server.Client().Transport, Host: server.URL} + + tags, err := client.Tags(ctx, "NOT USED!", "testrepo", "testimage") + assert.Nil(t, tags) + assert.Error(t, err) + assert.Contains(t, err.Error(), "unexpected image tags response") + }) + + t.Run("error on non-200 status code", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte("not found")) + })) + defer server.Close() + + client := &Client{ + Client: server.Client(), + log: log, + Options: Options{ + Token: "testtoken", + }, + } + client.Transport = &hostnameOverride{RT: server.Client().Transport, Host: server.URL} + + tags, err := client.Tags(ctx, "NOT USED!", "testrepo", "testimage") + assert.Nil(t, tags) + assert.Error(t, err) + assert.Contains(t, err.Error(), "unexpected image") + }) +} diff --git a/pkg/client/docker/types.go b/pkg/client/docker/types.go index bfeaa994..cd5ae877 100644 --- a/pkg/client/docker/types.go +++ b/pkg/client/docker/types.go @@ -12,9 +12,14 @@ type TagResponse struct { } type Result struct { - Name string `json:"name"` - Timestamp string `json:"last_updated"` - Images []Image `json:"images"` + Name string `json:"name"` + Timestamp string `json:"last_updated"` + TagStatus string `json:"tag_status"` // String of "active" or "inactive" + MediaType string `json:"media_type,omitempty"` + // Digest is only set with `application/vnd.oci.image.index.v1+json` media_type + Digest string `json:"digest,omitempty"` + + Images []Image `json:"images"` } type Image struct { diff --git a/pkg/client/ecr/ecr.go b/pkg/client/ecr/ecr.go index 3654fee4..49367d50 100644 --- a/pkg/client/ecr/ecr.go +++ b/pkg/client/ecr/ecr.go @@ -14,18 +14,20 @@ import ( "github.com/jetstack/version-checker/pkg/client/util" ) -type Client struct { - Config aws.Config +// Ensure that we are an ImageClient +var _ api.ImageClient = (*Client)(nil) +type Client struct { Options + Config aws.Config } type Options struct { + Transporter http.RoundTripper IamRoleArn string AccessKeyID string SecretAccessKey string SessionToken string - Transporter http.RoundTripper } func New(opts Options) *Client { @@ -64,28 +66,29 @@ func (c *Client) Tags(ctx context.Context, host, repo, image string) ([]api.Imag return nil, fmt.Errorf("failed to describe images: %s", err) } - var tags []api.ImageTag + tags := map[string]api.ImageTag{} for _, img := range images.ImageDetails { + // Base data shared across tags + base := api.ImageTag{ + SHA: *img.ImageDigest, + Timestamp: *img.ImagePushedAt, + } + // Continue early if no tags available if len(img.ImageTags) == 0 { - tags = append(tags, api.ImageTag{ - SHA: *img.ImageDigest, - Timestamp: *img.ImagePushedAt, - }) - + tags[base.SHA] = base continue } for _, tag := range img.ImageTags { - tags = append(tags, api.ImageTag{ - SHA: *img.ImageDigest, - Timestamp: *img.ImagePushedAt, - Tag: tag, - }) + current := base // copy the base + current.Tag = tag // set tag value + + util.BuildTags(tags, tag, ¤t) } } - return tags, nil + return util.TagMaptoList(tags), nil } func (c *Client) createClient(ctx context.Context, region string) (*ecr.Client, error) { diff --git a/pkg/client/fallback/fallback.go b/pkg/client/fallback/fallback.go index 12fc0c52..19748215 100644 --- a/pkg/client/fallback/fallback.go +++ b/pkg/client/fallback/fallback.go @@ -12,11 +12,13 @@ import ( "github.com/sirupsen/logrus" ) -type Client struct { - clients []api.ImageClient +// Ensure that we are an ImageClient +var _ api.ImageClient = (*Client)(nil) +type Client struct { log *logrus.Entry hostCache *cache.Cache + clients []api.ImageClient } func New(ctx context.Context, log *logrus.Entry, clients []api.ImageClient) (*Client, error) { diff --git a/pkg/client/gcr/gcr.go b/pkg/client/gcr/gcr.go index 15ebccc6..67eefaab 100644 --- a/pkg/client/gcr/gcr.go +++ b/pkg/client/gcr/gcr.go @@ -16,9 +16,12 @@ const ( lookupURL = "https://%s/v2/%s/tags/list" ) +// Ensure that we are an ImageClient +var _ api.ImageClient = (*Client)(nil) + type Options struct { - Token string Transporter http.RoundTripper + Token string } type Client struct { @@ -28,11 +31,12 @@ type Client struct { type Response struct { Manifest map[string]ManifestItem `json:"manifest"` + Tags []string `json:"tags,omitempty"` } type ManifestItem struct { - Tag []string `json:"tag"` TimeCreated string `json:"timeCreatedMs"` + Tags []string `json:"tag"` } func New(opts Options) *Client { @@ -98,24 +102,47 @@ func (c *Client) buildRequest(ctx context.Context, url string) (*http.Request, e } func (c *Client) extractImageTags(response Response) ([]api.ImageTag, error) { - var tags []api.ImageTag + tags := map[string]api.ImageTag{} for sha, manifestItem := range response.Manifest { timestamp, err := c.convertTimestamp(manifestItem.TimeCreated) if err != nil { return nil, fmt.Errorf("failed to convert timestamp string: %w", err) } + // Base data shared across tags + base := api.ImageTag{ + SHA: sha, + Timestamp: timestamp, + } + // If no tag, add without and continue early. - if len(manifestItem.Tag) == 0 { - tags = append(tags, api.ImageTag{SHA: sha, Timestamp: timestamp}) + if len(manifestItem.Tags) == 0 { + tags[sha] = base continue } - for _, tag := range manifestItem.Tag { - tags = append(tags, api.ImageTag{Tag: tag, SHA: sha, Timestamp: timestamp}) + for _, tag := range manifestItem.Tags { + current := base // copy the base + current.Tag = tag // set tag value + + // Already exists — add as child + if parent, exists := tags[tag]; exists { + parent.Children = append(parent.Children, ¤t) + tags[tag] = parent + } else { + // First occurrence — assign as root + tags[tag] = current + } } } - return tags, nil + + // Convert Map to Slice + taglist := make([]api.ImageTag, 0, len(tags)) + for _, tag := range tags { + taglist = append(taglist, tag) + } + + return taglist, nil } func (c *Client) convertTimestamp(timeCreated string) (time.Time, error) { diff --git a/pkg/client/gcr/path.go b/pkg/client/gcr/path.go index 69b678ea..7673e6ac 100644 --- a/pkg/client/gcr/path.go +++ b/pkg/client/gcr/path.go @@ -14,12 +14,16 @@ func (c *Client) IsHost(host string) bool { } func (c *Client) RepoImageFromPath(path string) (string, string) { - lastIndex := strings.LastIndex(path, "/") + split := strings.Split(path, "/") - // If there's no slash, then its a "root" level image - if lastIndex == -1 { - return "", path + lenSplit := len(split) + if lenSplit == 1 { + return "google-containers", split[0] } - return path[:lastIndex], path[lastIndex+1:] + if lenSplit > 1 { + return strings.Join(split[:len(split)-1], "/"), split[lenSplit-1] + } + + return path, "" } diff --git a/pkg/client/gcr/path_test.go b/pkg/client/gcr/path_test.go index 703c59a7..7bc80e47 100644 --- a/pkg/client/gcr/path_test.go +++ b/pkg/client/gcr/path_test.go @@ -1,6 +1,10 @@ package gcr -import "testing" +import ( + "testing" + + "github.com/stretchr/testify/assert" +) func TestIsHost(t *testing.T) { tests := map[string]struct { @@ -60,10 +64,9 @@ func TestIsHost(t *testing.T) { handler := new(Client) for name, test := range tests { t.Run(name, func(t *testing.T) { - if isHost := handler.IsHost(test.host); isHost != test.expIs { - t.Errorf("%s: unexpected IsHost, exp=%t got=%t", - test.host, test.expIs, isHost) - } + assert.Equal(t, test.expIs, + handler.IsHost(test.host), + ) }) } } @@ -94,10 +97,8 @@ func TestRepoImage(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { repo, image := handler.RepoImageFromPath(test.path) - if repo != test.expRepo && image != test.expImage { - t.Errorf("%s: unexpected repo/image, exp=%s/%s got=%s/%s", - test.path, test.expRepo, test.expImage, repo, image) - } + assert.Equal(t, test.expRepo, repo) + assert.Equal(t, test.expImage, image) }) } } diff --git a/pkg/client/ghcr/ghcr.go b/pkg/client/ghcr/ghcr.go index 40421d7e..80a2e5e2 100644 --- a/pkg/client/ghcr/ghcr.go +++ b/pkg/client/ghcr/ghcr.go @@ -13,16 +13,19 @@ import ( "github.com/google/go-github/v70/github" ) +// Ensure that we are an ImageClient +var _ api.ImageClient = (*Client)(nil) + type Options struct { + Transporter http.RoundTripper Token string Hostname string - Transporter http.RoundTripper } type Client struct { client *github.Client - opts Options ownerTypes map[string]string + opts Options } func New(opts Options) *Client { @@ -106,28 +109,43 @@ func (c *Client) buildPackageListOptions() *github.PackageListOptions { } func (c *Client) extractImageTags(versions []*github.PackageVersion) []api.ImageTag { - var tags []api.ImageTag + tags := map[string]api.ImageTag{} for _, ver := range versions { if meta, ok := ver.GetMetadata(); ok { - if len(meta.Container.Tags) == 0 { - continue - } - sha := "" + var sha string if strings.HasPrefix(*ver.Name, "sha") { sha = *ver.Name } + base := api.ImageTag{ + Tag: *ver.Name, + SHA: sha, + Timestamp: ver.CreatedAt.Time, + } + for _, tag := range meta.Container.Tags { - tags = append(tags, api.ImageTag{ - Tag: tag, - SHA: sha, - Timestamp: ver.CreatedAt.Time, - }) + current := base // copy the base + current.Tag = tag // set tag value + + // Tag Already exists — add as child + if parent, exists := tags[tag]; exists { + parent.Children = append(parent.Children, ¤t) + tags[tag] = parent + } else { + // First occurrence of Tag — assign as root + tags[tag] = current + } } } } - return tags + + // Convert Map to Slice + taglist := make([]api.ImageTag, 0, len(tags)) + for _, tag := range tags { + taglist = append(taglist, tag) + } + return taglist } func (c *Client) ownerType(ctx context.Context, owner string) (string, error) { diff --git a/pkg/client/ghcr/ghcr_test.go b/pkg/client/ghcr/ghcr_test.go index 39c2f6b6..ff9568f3 100644 --- a/pkg/client/ghcr/ghcr_test.go +++ b/pkg/client/ghcr/ghcr_test.go @@ -78,8 +78,7 @@ func TestClient_Tags(t *testing.T) { tags, err := client.Tags(ctx, host, "test-user-owner", "test-repo") assert.NoError(t, err) assert.Len(t, tags, 2) - assert.Equal(t, "tag1", tags[0].Tag) - assert.Equal(t, "tag2", tags[1].Tag) + assert.ElementsMatch(t, []string{"tag1", "tag2"}, []string{tags[0].Tag, tags[1].Tag}) }) t.Run("failed to fetch owner type", func(t *testing.T) { @@ -146,8 +145,7 @@ func TestClient_Tags(t *testing.T) { tags, err := client.Tags(ctx, host, "test-user-owner", "test-repo") assert.NoError(t, err) assert.Len(t, tags, 2) - assert.Equal(t, "tag1", tags[0].Tag) - assert.Equal(t, "tag2", tags[1].Tag) + assert.ElementsMatch(t, []string{"tag1", "tag2"}, []string{tags[0].Tag, tags[1].Tag}) }) t.Run("ownerType returns org", func(t *testing.T) { @@ -161,7 +159,6 @@ func TestClient_Tags(t *testing.T) { tags, err := client.Tags(ctx, host, "test-org-owner", "test-repo") assert.NoError(t, err) assert.Len(t, tags, 2) - assert.Equal(t, "tag1", tags[0].Tag) - assert.Equal(t, "tag2", tags[1].Tag) + assert.ElementsMatch(t, []string{"tag1", "tag2"}, []string{tags[0].Tag, tags[1].Tag}) }) } diff --git a/pkg/client/oci/helpers.go b/pkg/client/oci/helpers.go new file mode 100644 index 00000000..9bd1f909 --- /dev/null +++ b/pkg/client/oci/helpers.go @@ -0,0 +1,25 @@ +package oci + +import ( + "strings" + "time" +) + +const ( + CreatedTimeAnnotation = "org.opencontainers.image.created" + BuildDateAnnotation = "org.label-schema.build-date" +) + +func discoverTimestamp(annotations map[string]string) (timestamp time.Time, err error) { + if t, ok := annotations[CreatedTimeAnnotation]; ok { + timestamp, err = time.Parse(time.RFC3339, + strings.Replace(t, " ", "T", 1), + ) + } else if t, ok = annotations[BuildDateAnnotation]; ok { + timestamp, err = time.Parse(time.RFC3339, + strings.Replace(t, " ", "T", 1), + ) + } + + return timestamp, err +} diff --git a/pkg/client/oci/helpers_test.go b/pkg/client/oci/helpers_test.go new file mode 100644 index 00000000..407919af --- /dev/null +++ b/pkg/client/oci/helpers_test.go @@ -0,0 +1,77 @@ +package oci + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestDiscoverTimestamp(t *testing.T) { + tests := []struct { + name string + annotations map[string]string + expected time.Time + expectErr bool + }{ + { + name: "No annotations", + annotations: map[string]string{}, + expected: time.Time{}, + expectErr: false, + }, + { + name: "Valid CreatedTimeAnnotation", + annotations: map[string]string{ + CreatedTimeAnnotation: "2023-03-15T12:34:56Z", + }, + expected: time.Date(2023, 3, 15, 12, 34, 56, 0, time.UTC), + expectErr: false, + }, + { + name: "Valid BuildDateAnnotation", + annotations: map[string]string{ + BuildDateAnnotation: "2023-03-15T12:34:56Z", + }, + expected: time.Date(2023, 3, 15, 12, 34, 56, 0, time.UTC), + expectErr: false, + }, + { + name: "Invalid CreatedTimeAnnotation format", + annotations: map[string]string{ + CreatedTimeAnnotation: "invalid-date", + }, + expected: time.Time{}, + expectErr: true, + }, + { + name: "Invalid BuildDateAnnotation format", + annotations: map[string]string{ + BuildDateAnnotation: "invalid-date", + }, + expected: time.Time{}, + expectErr: true, + }, + { + name: "Both annotations present, prefer CreatedTimeAnnotation", + annotations: map[string]string{ + CreatedTimeAnnotation: "2023-03-15T12:34:56Z", + BuildDateAnnotation: "2023-01-01T00:00:00Z", + }, + expected: time.Date(2023, 3, 15, 12, 34, 56, 0, time.UTC), + expectErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := discoverTimestamp(tt.annotations) + if tt.expectErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.expected, result) + } + }) + } +} diff --git a/pkg/client/oci/oci.go b/pkg/client/oci/oci.go index e67ed669..3ab51fcb 100644 --- a/pkg/client/oci/oci.go +++ b/pkg/client/oci/oci.go @@ -6,14 +6,20 @@ import ( "net/http" "runtime" "strings" + "sync" + + "github.com/sirupsen/logrus" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/types" "github.com/jetstack/version-checker/pkg/api" ) +var numWorkers = runtime.NumCPU() * 5 + type Options struct { Transporter http.RoundTripper Auth *authn.AuthConfig @@ -29,14 +35,18 @@ func (o *Options) Authorization() (*authn.AuthConfig, error) { // Client is a client for a registry compatible with the OCI Distribution Spec type Client struct { *Options + log *logrus.Entry puller *remote.Puller } +// Ensure that we are an ImageClient +var _ api.ImageClient = (*Client)(nil) + // New returns a new client -func New(opts *Options) (*Client, error) { +func New(opts *Options, log *logrus.Entry) (*Client, error) { pullOpts := []remote.Option{ - remote.WithJobs(runtime.NumCPU()), - remote.WithUserAgent("version-checker"), + remote.WithJobs(numWorkers), + remote.WithUserAgent("version-checker/oci"), remote.WithAuth(opts), } if opts.Transporter != nil { @@ -50,6 +60,7 @@ func New(opts *Options) (*Client, error) { return &Client{ puller: puller, + log: log.WithField("client", "OCI"), Options: opts, }, nil } @@ -70,13 +81,98 @@ func (c *Client) Tags(ctx context.Context, host, repo, image string) ([]api.Imag if err != nil { return nil, fmt.Errorf("listing tags: %w", err) } + c.log.Infof("Collected %v tags..", len(bareTags)) + return c.Manifests(ctx, reg.Repo(repo, image), bareTags) +} - var tags []api.ImageTag - for _, t := range bareTags { - tags = append(tags, api.ImageTag{Tag: t}) +func (c *Client) Manifests(ctx context.Context, repo name.Repository, tags []string) (fulltags []api.ImageTag, err error) { + wg := sync.WaitGroup{} + sem := make(chan struct{}, numWorkers) // limit concurrent fetches + wg.Add(len(tags)) + mu := sync.Mutex{} + + // Lets lookup all the child Manifests (where applicable) + for _, tag := range tags { + go func(repo name.Repository, tag string) { + log := c.log.WithFields(logrus.Fields{"tag": tag, "repo": repo.Name()}) + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + + // Parse the Tag + t, err := name.NewTag(repo.Name() + ":" + tag) + if err != nil { + log.Errorf("parsing Tag: %s", err) + return + } + + // Fetch the manifest + manifest, err := c.puller.Get(ctx, t) + if err != nil { + log.Errorf("getting manifest: %s", err) + return + } + + // Lock when we have the data! + mu.Lock() + defer mu.Unlock() + + ts, err := discoverTimestamp(manifest.Annotations) + if err != nil { + log.Errorf("Unable to discover Timestamp: %s", err) + return + } + + baseTag := api.ImageTag{ + Tag: tag, + Timestamp: ts, + } + + switch manifest.MediaType { + + case types.OCIImageIndex, types.DockerManifestList: + children := []*api.ImageTag{} + imgidx, err := manifest.ImageIndex() + if err != nil { + log.Errorf("getting ImageIndex: %s", err) + return + } + idxman, err := imgidx.IndexManifest() + if err != nil { + log.Errorf("getting IndexManifest: %s", err) + return + } + for _, img := range idxman.Manifests { + + children = append(children, &api.ImageTag{ + Tag: tag, + SHA: img.Digest.String(), + }) + } + baseTag.Children = children + + case types.OCIManifestSchema1, types.DockerManifestSchema2: + img, err := manifest.Image() + if err != nil { + log.Errorf("unable to collect image from manifest: %s", err) + return + } + sha, err := img.Digest() + if err != nil { + log.Errorf("unable to collect digest from manifest: %s", err) + return + } + baseTag.SHA = sha.String() + } + + // Add it to the full tags + fulltags = append(fulltags, baseTag) + }(repo, tag) } + // Wait for everything to complete! + wg.Wait() - return tags, nil + return fulltags, err } // IsHost always returns true because it supports any host @@ -94,7 +190,7 @@ func (c *Client) RepoImageFromPath(path string) (string, string) { } if lenSplit > 1 { - return split[lenSplit-2], split[lenSplit-1] + return strings.Join(split[:len(split)-1], "/"), split[lenSplit-1] } return path, "" diff --git a/pkg/client/oci/oci_test.go b/pkg/client/oci/oci_test.go index f21a2e17..35767107 100644 --- a/pkg/client/oci/oci_test.go +++ b/pkg/client/oci/oci_test.go @@ -3,11 +3,16 @@ package oci import ( "context" "fmt" + "io" + "log" "net/http/httptest" "net/url" "testing" - "github.com/google/go-cmp/cmp" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/registry" "github.com/google/go-containerregistry/pkg/v1/empty" @@ -17,6 +22,8 @@ import ( func TestClientTags(t *testing.T) { ctx := context.Background() + emptySha, err := empty.Image.Digest() + require.NoError(t, err) type testCase struct { repo string @@ -32,23 +39,25 @@ func TestClientTags(t *testing.T) { wantTags: []api.ImageTag{ { Tag: "a", + SHA: emptySha.String(), }, { Tag: "b", + SHA: emptySha.String(), }, { Tag: "c", + SHA: emptySha.String(), }, }, } repo, err := name.NewRepository(fmt.Sprintf("%s/%s/%s", host, tc.repo, tc.img)) - if err != nil { - t.Fatalf("unexpected error parsing repo: %s", err) - } + require.NoError(t, err) + for _, tag := range tc.wantTags { - if err := remote.Write(repo.Tag(tag.Tag), empty.Image); err != nil { - t.Fatalf("unexpected error writing image to tag: %s", err) - } + require.NoError(t, + remote.Write(repo.Tag(tag.Tag), empty.Image), + ) } return tc }, @@ -58,20 +67,21 @@ func TestClientTags(t *testing.T) { wantTags: []api.ImageTag{ { Tag: "a", + SHA: emptySha.String(), }, { Tag: "b", + SHA: emptySha.String(), }, }, } repo, err := name.NewRepository(fmt.Sprintf("%s/%s", host, tc.img)) - if err != nil { - t.Fatalf("unexpected error parsing repo: %s", err) - } + require.NoError(t, err) + for _, tag := range tc.wantTags { - if err := remote.Write(repo.Tag(tag.Tag), empty.Image); err != nil { - t.Fatalf("unexpected error writing image to tag: %s", err) - } + require.NoError(t, + remote.Write(repo.Tag(tag.Tag), empty.Image), + ) } return tc }, @@ -82,17 +92,17 @@ func TestClientTags(t *testing.T) { wantTags: []api.ImageTag{ { Tag: "a", + SHA: emptySha.String(), }, }, } repo, err := name.NewRepository(fmt.Sprintf("%s/%s/%s", host, tc.repo, tc.img)) - if err != nil { - t.Fatalf("unexpected error parsing repo: %s", err) - } + require.NoError(t, err) + for _, tag := range tc.wantTags { - if err := remote.Write(repo.Tag(tag.Tag), empty.Image); err != nil { - t.Fatalf("unexpected error writing image to tag: %s", err) - } + require.NoError(t, + remote.Write(repo.Tag(tag.Tag), empty.Image), + ) } return tc }, @@ -102,18 +112,16 @@ func TestClientTags(t *testing.T) { img: "bar", } repo, err := name.NewRepository(fmt.Sprintf("%s/%s/%s", host, tc.repo, tc.img)) - if err != nil { - t.Fatalf("unexpected error parsing repo: %s", err) - } + require.NoError(t, err) // Write a tag but then delete it so the repository // exists but it has no tags - if err := remote.Write(repo.Tag("latest"), empty.Image); err != nil { - t.Fatalf("unexpected error writing image to tag: %s", err) - } - if err := remote.Delete(repo.Tag("latest")); err != nil { - t.Fatalf("unexpected error writing image to tag: %s", err) - } + require.NoError(t, + remote.Write(repo.Tag("latest"), empty.Image), + ) + require.NoError(t, + remote.Delete(repo.Tag("latest")), + ) return tc }, "should return an error when listing a repository that doesn't exist": func(t *testing.T, host string) *testCase { @@ -129,23 +137,21 @@ func TestClientTags(t *testing.T) { t.Run(testName, func(t *testing.T) { host := setupRegistry(t) - c, err := New(new(Options)) - if err != nil { - t.Fatalf("unexpected error creating client: %s", err) - } + c, err := New(new(Options), logrus.NewEntry(logrus.New())) + require.NoError(t, err) tc := fn(t, host) gotTags, err := c.Tags(ctx, host, tc.repo, tc.img) - if tc.wantErr && err == nil { - t.Errorf("unexpected nil error listing tags") - } - if !tc.wantErr && err != nil { - t.Errorf("unexpected error listing tags: %s", err) - } - if diff := cmp.Diff(tc.wantTags, gotTags); diff != "" { - t.Errorf("unexpected tags:\n%s", diff) - } + + if tc.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + // We don't care about the order - but to ensure that the elements we expect + // exist within the output + assert.ElementsMatch(t, tc.wantTags, gotTags) }) } } @@ -177,27 +183,27 @@ func TestClientRepoImageFromPath(t *testing.T) { }, } - c, err := New(new(Options)) + c, err := New(new(Options), logrus.NewEntry(logrus.New())) if err != nil { t.Fatalf("unexpected error creating client: %s", err) } for name, test := range tests { t.Run(name, func(t *testing.T) { repo, image := c.RepoImageFromPath(test.path) - if repo != test.expRepo && image != test.expImage { - t.Errorf("%s: unexpected repo/image, exp=%s/%s got=%s/%s", - test.path, test.expRepo, test.expImage, repo, image) - } + assert.Equal(t, test.expRepo, repo) + assert.Equal(t, test.expImage, image) }) } } func setupRegistry(t *testing.T) string { - r := httptest.NewServer(registry.New()) + r := httptest.NewServer(registry.New( + registry.Logger(log.New(io.Discard, "", log.LstdFlags)), + registry.WithReferrersSupport(false), + )) t.Cleanup(r.Close) u, err := url.Parse(r.URL) - if err != nil { - t.Fatalf("unexpected error parsing registry url: %s", err) - } + require.NoError(t, err) + return u.Host } diff --git a/pkg/client/quay/api_types.go b/pkg/client/quay/api_types.go new file mode 100644 index 00000000..fc87b091 --- /dev/null +++ b/pkg/client/quay/api_types.go @@ -0,0 +1,35 @@ +package quay + +import ( + "github.com/jetstack/version-checker/pkg/api" +) + +type responseTag struct { + Tags []responseTagItem `json:"tags"` + HasAdditional bool `json:"has_additional"` + Page int `json:"page"` +} + +type responseTagItem struct { + Name string `json:"name"` + ManifestDigest string `json:"manifest_digest"` + LastModified string `json:"last_modified"` + IsManifestList bool `json:"is_manifest_list"` +} + +type responseManifest struct { + Status *int `json:"status,omitempty"` + ManifestData string `json:"manifest_data"` +} + +type responseManifestData struct { + Manifests []responseManifestDataItem `json:"manifests"` +} + +type responseManifestDataItem struct { + Digest string `json:"digest"` + Platform struct { + Architecture api.Architecture `json:"architecture"` + OS api.OS `json:"os"` + } `json:"platform"` +} diff --git a/pkg/client/quay/pager.go b/pkg/client/quay/pager.go index ef3ce7b6..c99a81f7 100644 --- a/pkg/client/quay/pager.go +++ b/pkg/client/quay/pager.go @@ -14,11 +14,11 @@ type pager struct { repo, image string - mu sync.Mutex - wg sync.WaitGroup - tags []api.ImageTag errs []error + wg sync.WaitGroup + + mu sync.Mutex } func (c *Client) newPager(repo, image string) *pager { @@ -55,20 +55,28 @@ func (p *pager) fetchTags(ctx context.Context) error { // fetchTagsPaged will return the image tags from a given page number, as well // as if there are more pages. func (p *pager) fetchTagsPaged(ctx context.Context, page int) (bool, error) { + select { + case <-ctx.Done(): + return false, ctx.Err() + default: + } + url := fmt.Sprintf(tagURL, p.repo, p.image, page) var resp responseTag if err := p.makeRequest(ctx, url, &resp); err != nil { return false, err } + sem := make(chan struct{}, 10) // limit concurrent fetches p.wg.Add(len(resp.Tags)) - // Concurrently fetch all images from a given tag - for i := range resp.Tags { - go func(i int) { + for _, tag := range resp.Tags { + go func(tag responseTagItem) { defer p.wg.Done() + sem <- struct{}{} + defer func() { <-sem }() - imageTags, err := p.fetchImageManifest(ctx, p.repo, p.image, &resp.Tags[i]) + imageTag, err := p.fetchImageManifest(ctx, p.repo, p.image, &tag) p.mu.Lock() defer p.mu.Unlock() @@ -78,8 +86,8 @@ func (p *pager) fetchTagsPaged(ctx context.Context, page int) (bool, error) { return } - p.tags = append(p.tags, imageTags...) - }(i) + p.tags = append(p.tags, *imageTag) + }(tag) } return resp.HasAdditional, nil diff --git a/pkg/client/quay/path_test.go b/pkg/client/quay/path_test.go index 5fff08e1..8cf19b8b 100644 --- a/pkg/client/quay/path_test.go +++ b/pkg/client/quay/path_test.go @@ -1,6 +1,10 @@ package quay -import "testing" +import ( + "testing" + + "github.com/stretchr/testify/assert" +) func TestIsHost(t *testing.T) { tests := map[string]struct { @@ -40,10 +44,9 @@ func TestIsHost(t *testing.T) { handler := new(Client) for name, test := range tests { t.Run(name, func(t *testing.T) { - if isHost := handler.IsHost(test.host); isHost != test.expIs { - t.Errorf("%s: unexpected IsHost, exp=%t got=%t", - test.host, test.expIs, isHost) - } + assert.Equal(t, test.expIs, + handler.IsHost(test.host), + ) }) } } @@ -74,10 +77,8 @@ func TestRepoImage(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { repo, image := handler.RepoImageFromPath(test.path) - if repo != test.expRepo && image != test.expImage { - t.Errorf("%s: unexpected repo/image, exp=%s/%s got=%s/%s", - test.path, test.expRepo, test.expImage, repo, image) - } + assert.Equal(t, test.expRepo, repo) + assert.Equal(t, test.expImage, image) }) } } diff --git a/pkg/client/quay/quay.go b/pkg/client/quay/quay.go index 7130084a..58dcd395 100644 --- a/pkg/client/quay/quay.go +++ b/pkg/client/quay/quay.go @@ -20,8 +20,8 @@ const ( ) type Options struct { - Token string Transporter http.RoundTripper + Token string } type Client struct { @@ -29,35 +29,8 @@ type Client struct { Options } -type responseTag struct { - Tags []responseTagItem `json:"tags"` - HasAdditional bool `json:"has_additional"` - Page int `json:"page"` -} - -type responseTagItem struct { - Name string `json:"name"` - ManifestDigest string `json:"manifest_digest"` - LastModified string `json:"last_modified"` - IsManifestList bool `json:"is_manifest_list"` -} - -type responseManifest struct { - ManifestData string `json:"manifest_data"` - Status *int `json:"status,omitempty"` -} - -type responseManifestData struct { - Manifests []responseManifestDataItem `json:"manifests"` -} - -type responseManifestDataItem struct { - Digest string `json:"digest"` - Platform struct { - Architecture api.Architecture `json:"architecture"` - OS api.OS `json:"os"` - } `json:"platform"` -} +// Ensure that we are an ImageClient +var _ api.ImageClient = (*Client)(nil) func New(opts Options, log *logrus.Entry) *Client { client := retryablehttp.NewClient() @@ -89,60 +62,58 @@ func (c *Client) Tags(ctx context.Context, _, repo, image string) ([]api.ImageTa } // fetchImageManifest will lookup all manifests for a tag, if it is a list. -func (c *Client) fetchImageManifest(ctx context.Context, repo, image string, tag *responseTagItem) ([]api.ImageTag, error) { +func (c *Client) fetchImageManifest(ctx context.Context, repo, image string, tag *responseTagItem) (*api.ImageTag, error) { timestamp, err := time.Parse(time.RFC1123Z, tag.LastModified) if err != nil { return nil, err } + iTag := &api.ImageTag{ + Tag: tag.Name, + SHA: tag.ManifestDigest, + Timestamp: timestamp, + OS: "", + Architecture: "", + } + // If a multi-arch image, call manifest endpoint if tag.IsManifestList { url := fmt.Sprintf(manifestURL, repo, image, tag.ManifestDigest) - tags, err := c.callManifests(ctx, timestamp, tag.Name, url) + err := c.callManifests(ctx, timestamp, iTag, url) if err != nil { return nil, err } - return tags, nil + return iTag, nil } // Fallback to not using multi-arch image + iTag.OS, iTag.Architecture = util.OSArchFromTag(tag.Name) - os, arch := util.OSArchFromTag(tag.Name) - - return []api.ImageTag{ - { - Tag: tag.Name, - SHA: tag.ManifestDigest, - Timestamp: timestamp, - OS: os, - Architecture: arch, - }, - }, nil + return iTag, nil } // callManifests endpoint on the tags image manifest. -func (c *Client) callManifests(ctx context.Context, timestamp time.Time, tag, url string) ([]api.ImageTag, error) { +func (c *Client) callManifests(ctx context.Context, timestamp time.Time, tag *api.ImageTag, url string) error { var manifestResp responseManifest if err := c.makeRequest(ctx, url, &manifestResp); err != nil { - return nil, err + return err } // Got error on this manifest, ignore if manifestResp.Status != nil { - return nil, nil + return nil } var manifestData responseManifestData if err := json.Unmarshal([]byte(manifestResp.ManifestData), &manifestData); err != nil { - return nil, fmt.Errorf("failed to unmarshal manifest data %s: %#+v: %s", - tag, manifestResp, err) + return fmt.Errorf("failed to unmarshal manifest data %s: %#+v: %s", + tag.Tag, manifestResp, err) } - var tags []api.ImageTag for _, manifest := range manifestData.Manifests { - tags = append(tags, api.ImageTag{ - Tag: tag, + tag.Children = append(tag.Children, &api.ImageTag{ + Tag: tag.Tag, SHA: manifest.Digest, Timestamp: timestamp, Architecture: manifest.Platform.Architecture, @@ -150,7 +121,7 @@ func (c *Client) callManifests(ctx context.Context, timestamp time.Time, tag, ur }) } - return tags, nil + return nil } // makeRequest will make a call and write the response to the object. diff --git a/pkg/client/selfhosted/api_types.go b/pkg/client/selfhosted/api_types.go index af205d2c..66ef4024 100644 --- a/pkg/client/selfhosted/api_types.go +++ b/pkg/client/selfhosted/api_types.go @@ -1,6 +1,7 @@ package selfhosted import ( + "encoding/json" "time" "github.com/jetstack/version-checker/pkg/api" @@ -15,15 +16,60 @@ type TagResponse struct { } type ManifestResponse struct { - Digest string + Digest string `json:"digest,omitempty"` Architecture api.Architecture `json:"architecture"` History []History `json:"history"` } +type ManafestListResponse struct { + Manifests []ManifestResponse `json:"manifests"` +} + type History struct { - V1Compatibility string `json:"v1Compatibility"` + V1Compatibility V1CompatibilityWrapper `json:"v1Compatibility"` } type V1Compatibility struct { Created time.Time `json:"created,omitempty"` } + +type V1CompatibilityWrapper struct { + V1Compatibility +} + +func (v *V1CompatibilityWrapper) UnmarshalJSON(b []byte) error { + var raw string + if err := json.Unmarshal(b, &raw); err != nil { + return err + } + return json.Unmarshal([]byte(raw), &v.V1Compatibility) +} + +func (v V1CompatibilityWrapper) MarshalJSON() ([]byte, error) { + marshaled, err := json.Marshal(v.V1Compatibility) + if err != nil { + return nil, err + } + return json.Marshal(string(marshaled)) // ← Double encode: inner to string +} + +type ErrorResponse struct { + Errors []ErrorType `json:"errors"` +} + +type ErrorType struct { + Code string `json:"code"` + Message string `json:"message"` +} + +type V2ManifestListResponse struct { + SchemaVersion int `json:"schemaVersion"` + MediaType string `json:"mediaType"` + Manifests []V2ManifestListEntry `json:"manifests"` +} + +type V2ManifestListEntry struct { + Digest string `json:"digest"` + MediaType string `json:"mediaType"` + Platform api.Platform `json:"platform"` +} diff --git a/pkg/client/selfhosted/path.go b/pkg/client/selfhosted/path.go index 3e444c00..9d99b049 100644 --- a/pkg/client/selfhosted/path.go +++ b/pkg/client/selfhosted/path.go @@ -13,6 +13,9 @@ const ( ) func (c *Client) IsHost(host string) bool { + if c.hostRegex == nil { + return c.Host == host + } return c.hostRegex.MatchString(host) } diff --git a/pkg/client/selfhosted/path_test.go b/pkg/client/selfhosted/path_test.go index 6824427f..0df5684a 100644 --- a/pkg/client/selfhosted/path_test.go +++ b/pkg/client/selfhosted/path_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" ) func TestIsHost(t *testing.T) { @@ -77,12 +78,28 @@ func TestIsHost(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { - if isHost := handler.IsHost(test.host); isHost != test.expIs { - t.Errorf("%s: unexpected IsHost, exp=%t got=%t", - test.host, test.expIs, isHost) - } + assert.Equal(t, test.expIs, + handler.IsHost(test.host), + ) + assert.Equal(t, test.expIs, handler.IsHost(test.host)) }) } + + // t.Run("No Options set on client", func(t *testing.T) { + // handler, err := New(context.TODO(), logrus.NewEntry(logrus.New()), &Options{}) + // require.NoError(t, err) + + // assert.NotPanics(t, func() { handler.IsHost("example.com") }) + // }) + + // t.Run("No Options set on client", func(t *testing.T) { + + // handler := &Client{Options: &Options{Host: "abc"}} + // require.NoError(t, err) + + // assert.NotPanics(t, func() { handler.IsHost("example.com") }) + // assert.True(t, handler.IsHost("abc")) + // }) } func TestRepoImage(t *testing.T) { diff --git a/pkg/client/selfhosted/selfhosted.go b/pkg/client/selfhosted/selfhosted.go index 47982684..ef475e16 100644 --- a/pkg/client/selfhosted/selfhosted.go +++ b/pkg/client/selfhosted/selfhosted.go @@ -24,6 +24,9 @@ import ( "github.com/jetstack/version-checker/pkg/client/util" ) +// Ensure that we are an ImageClient +var _ api.ImageClient = (*Client)(nil) + const ( // {host}/v2/{repo/image}/tags/list?n=500 tagsPath = "%s/v2/%s/tags/list?n=500" @@ -33,21 +36,22 @@ const ( defaultTokenPath = "/v2/token" // HTTP headers to request API version - dockerAPIv1Header = "application/vnd.docker.distribution.manifest.v1+json" - dockerAPIv2Header = "application/vnd.docker.distribution.manifest.v2+json" + dockerAPIv1Header = "application/vnd.docker.distribution.manifest.v1+json" + dockerAPIv2Header = "application/vnd.docker.distribution.manifest.v2+json" + dockerAPIv2ManifestList = "application/vnd.docker.distribution.manifest.list.v2+json" ) type Options struct { - Host string - Username string - Password string - Bearer string - TokenPath string - Insecure bool - CAPath string Transporter http.RoundTripper -} + Host string + Username string + Password string + Bearer string + TokenPath string + CAPath string + Insecure bool +} type Client struct { *http.Client *Options @@ -169,7 +173,7 @@ func (c *Client) Tags(ctx context.Context, host, repo, image string) ([]api.Imag return nil, err } - var tags []api.ImageTag + tags := map[string]api.ImageTag{} for _, tag := range tagResponse.Tags { manifestURL := fmt.Sprintf(manifestPath, host, path, tag) @@ -187,20 +191,16 @@ func (c *Client) Tags(ctx context.Context, host, repo, image string) ([]api.Imag var timestamp time.Time for _, v1History := range manifestResponse.History { - data := V1Compatibility{} - if err := json.Unmarshal([]byte(v1History.V1Compatibility), &data); err != nil { - return nil, err - } - - if !data.Created.IsZero() { - timestamp = data.Created + if !v1History.V1Compatibility.Created.IsZero() { + timestamp = v1History.V1Compatibility.Created // Each layer has its own created timestamp. We just want a general reference. // Take the first and step out the loop break } } - header, err := c.doRequest(ctx, manifestURL, dockerAPIv2Header, new(ManifestResponse)) + var manifestListResponse V2ManifestListResponse + header, err := c.doRequest(ctx, manifestURL, strings.Join([]string{dockerAPIv2Header, dockerAPIv2ManifestList}, ","), &manifestListResponse) if httpErr, ok := selfhostederrors.IsHTTPError(err); ok { c.log.Errorf("%s: failed to get manifest sha response for tag, skipping (%d): %s", manifestURL, httpErr.StatusCode, httpErr.Body) @@ -210,15 +210,28 @@ func (c *Client) Tags(ctx context.Context, host, repo, image string) ([]api.Imag return nil, err } - tags = append(tags, api.ImageTag{ + // Lets set as much of the current as we know + current := api.ImageTag{ Tag: tag, SHA: header.Get("Docker-Content-Digest"), Timestamp: timestamp, - Architecture: manifestResponse.Architecture, - }) - } + Architecture: api.Architecture(manifestResponse.Architecture), + } - return tags, nil + util.BuildTags(tags, tag, ¤t) + + for _, manifest := range manifestListResponse.Manifests { + + // If we didn't get a SHA from the inital call, + // lets set it from the manifestList + if current.SHA == "" && manifest.Digest != "" { + current.SHA = manifest.Digest + } + + util.BuildTags(tags, tag, ¤t) + } + } + return util.TagMaptoList(tags), nil } func (c *Client) doRequest(ctx context.Context, url, header string, obj interface{}) (http.Header, error) { @@ -227,6 +240,7 @@ func (c *Client) doRequest(ctx context.Context, url, header string, obj interfac if err != nil { return nil, err } + req.Header.Set("User-Agent", "version-checker/selfhosted") req = req.WithContext(ctx) if len(c.Bearer) > 0 { @@ -256,7 +270,7 @@ func (c *Client) doRequest(ctx context.Context, url, header string, obj interfac } if err := json.Unmarshal(body, obj); err != nil { - return nil, fmt.Errorf("unexpected %s response: %s", url, body) + return nil, fmt.Errorf("unexpected %s response: %s - %w", url, body, err) } return resp.Header, nil diff --git a/pkg/client/selfhosted/selfhosted_test.go b/pkg/client/selfhosted/selfhosted_test.go index 223e6e6b..d9144130 100644 --- a/pkg/client/selfhosted/selfhosted_test.go +++ b/pkg/client/selfhosted/selfhosted_test.go @@ -3,16 +3,20 @@ package selfhosted import ( "context" "encoding/base64" + "encoding/json" "errors" "fmt" "net/http" "net/http/httptest" "net/url" "os" + "strings" "testing" + "time" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/jetstack/version-checker/pkg/api" selfhostederrors "github.com/jetstack/version-checker/pkg/client/selfhosted/errors" @@ -126,17 +130,70 @@ func TestTags(t *testing.T) { } server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // l.Infof("Got request: %v", r) switch r.URL.Path { case "/v2/repo/image/tags/list": w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(`{"tags":["v1.0.0","v2.0.0"]}`)) + _ = json.NewEncoder(w).Encode(TagResponse{Tags: []string{"v1.0.0", "v2.0.0"}}) + case "/v2/repo/image/manifests/v1.0.0": w.Header().Add("Docker-Content-Digest", "sha256:abcdef") w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(`{"architecture":"amd64","history":[{"v1Compatibility":"{\"created\":\"2023-08-27T12:00:00Z\"}"}]}`)) + _ = json.NewEncoder(w).Encode(ManifestResponse{ + Architecture: api.Architecture("amd64"), + History: []History{ + { + V1Compatibility: V1CompatibilityWrapper{ + V1Compatibility: V1Compatibility{Created: time.Now()}, + }, + }, + }, + }) + case "/v2/repo/image/manifests/v2.0.0": w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(`{}`)) // Write some blank content + + // This image is a manifest List + case "/v2/repo/multiimage/tags/list": + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(TagResponse{Tags: []string{"v2.2.0"}}) + + case "/v2/repo/multiimage/manifests/v2.2.0": + acpt := r.Header.Get("Accept") + log.Warnf("Got following request: %v", acpt) + switch acpt { + // If we have multiple formats... + case strings.Join([]string{dockerAPIv2Header, dockerAPIv2ManifestList}, ","): + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(V2ManifestListResponse{ + Manifests: []V2ManifestListEntry{ + {Digest: "asjhfvbasjhbfsaj", Platform: api.Platform{OS: api.OS("Linux"), Architecture: api.Architecture("arm64")}}, + }, + }) + + // Docker V1 API + case dockerAPIv1Header: + w.WriteHeader(http.StatusOK) + w.Header().Add("Docker-Content-Digest", "sha265:asgjnaskjgbsajgsa") + _, _ = w.Write([]byte(`{}`)) // Write some blank content + + // Docker V2 Header... + case dockerAPIv2Header: + w.WriteHeader(http.StatusNotFound) + _ = json.NewEncoder(w).Encode(ErrorResponse{Errors: []ErrorType{ + { + Code: "MANIFEST_UNKNOWN", + Message: `Manifest has media type "application/vnd.docker.distribution.manifest.list.v2+json" but client accepts ["application/vnd.docker.distribution.manifest.v1+json"]`, + }, + }}) + + // ManifestList ONLY + case dockerAPIv2ManifestList: + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{}`)) // Write some blank content + + } } })) defer server.Close() @@ -144,14 +201,24 @@ func TestTags(t *testing.T) { h, err := url.Parse(server.URL) assert.NoError(t, err) - tags, err := client.Tags(ctx, h.Host, "repo", "image") + t.Run("Standard Single Arch Image", func(t *testing.T) { + tags, err := client.Tags(ctx, h.Host, "repo", "image") + require.NoError(t, err) + require.Len(t, tags, 2) - assert.NoError(t, err) - assert.Len(t, tags, 2) - assert.Equal(t, "v1.0.0", tags[0].Tag) - assert.Equal(t, api.Architecture("amd64"), tags[0].Architecture) - assert.Equal(t, "sha256:abcdef", tags[0].SHA) - assert.Equal(t, "v2.0.0", tags[1].Tag) + // We don't care of the order, we just want to make sure we have the tags + assert.ElementsMatch(t, []string{"v1.0.0", "v2.0.0"}, []string{tags[0].Tag, tags[1].Tag}) + assert.Equal(t, api.Architecture("amd64"), tags[0].Architecture) + assert.Equal(t, "sha256:abcdef", tags[0].SHA) + }) + + t.Run("MultiArch ManifestList v2.2", func(t *testing.T) { + tags, err := client.Tags(ctx, h.Host, "repo", "multiimage") + + assert.NoError(t, err) + require.Len(t, tags, 1) + assert.Equal(t, "v2.2.0", tags[0].Tag) + }) }) t.Run("error fetching tags", func(t *testing.T) { diff --git a/pkg/client/util/http_backoff_limiter.go b/pkg/client/util/http_backoff_limiter.go new file mode 100644 index 00000000..02b50b8c --- /dev/null +++ b/pkg/client/util/http_backoff_limiter.go @@ -0,0 +1,49 @@ +package util + +import ( + "fmt" + "net/http" + "time" + + "github.com/sirupsen/logrus" + + "github.com/hashicorp/go-retryablehttp" + "golang.org/x/time/rate" +) + +func RateLimitedBackoffLimiter( + logger *logrus.Entry, + limiter *rate.Limiter, + maxWait time.Duration, +) retryablehttp.Backoff { + return func(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration { + + defaultDelay := retryablehttp.DefaultBackoff(min, max, attemptNum, resp) + // Reserve first to introspect delay + res := limiter.Reserve() + if !res.OK() { + logger.Error(fmt.Errorf("rate limit exceeded"), "Cannot make request") + return maxWait // fallback + } + rateDelay := res.Delay() + + // Choose the larger of the two delays (rate limit or default backoff) + delay := defaultDelay + if rateDelay > delay { + delay = rateDelay + } + + if delay > maxWait { + res.Cancel() + logger.WithFields(logrus.Fields{ + "attempt": attemptNum, "wait": delay, "maxWait": maxWait, + }).Info("Wait time too long, using max wait instead") + return maxWait + } + + logger.WithFields(logrus.Fields{ + "attempt": attemptNum, "wait": delay, + }).Info("Waiting due to rate limit") + return delay + } +} diff --git a/pkg/client/util/http_backoff_limiter_test.go b/pkg/client/util/http_backoff_limiter_test.go new file mode 100644 index 00000000..1a553c31 --- /dev/null +++ b/pkg/client/util/http_backoff_limiter_test.go @@ -0,0 +1,65 @@ +package util + +import ( + "testing" + "time" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "golang.org/x/time/rate" +) + +func TestRateLimitedBackoffLimiter(t *testing.T) { + logger := logrus.NewEntry(logrus.New()) + maxWait := 5 * time.Second + + t.Run("default backoff delay is used when rate limiter allows immediate execution", func(t *testing.T) { + limiter := rate.NewLimiter(rate.Every(1*time.Millisecond), 1) + backoff := RateLimitedBackoffLimiter(logger, limiter, maxWait) + + min := 100 * time.Millisecond + max := 200 * time.Millisecond + attemptNum := 1 + + delay := backoff(min, max, attemptNum, nil) + assert.GreaterOrEqual(t, delay, min) + assert.LessOrEqual(t, delay, max) + }) + + t.Run("rate limiter delay is used when it exceeds default backoff", func(t *testing.T) { + limiter := rate.NewLimiter(rate.Every(500*time.Millisecond), 1) + backoff := RateLimitedBackoffLimiter(logger, limiter, maxWait) + + min := 100 * time.Millisecond + max := 500 * time.Millisecond + attemptNum := 3 + + delay := backoff(min, max, attemptNum, nil) + assert.GreaterOrEqual(t, delay, 500*time.Millisecond) + assert.LessOrEqual(t, delay, maxWait) + }) + + t.Run("maxWait is used when delay exceeds maxWait", func(t *testing.T) { + limiter := rate.NewLimiter(rate.Every(10*time.Second), 1) + backoff := RateLimitedBackoffLimiter(logger, limiter, maxWait) + + min := maxWait - time.Second + max := maxWait + attemptNum := 3 + + delay := backoff(min, max, attemptNum, nil) + assert.Equal(t, maxWait, delay) + }) + + t.Run("rate limiter reservation fails", func(t *testing.T) { + limiter := rate.NewLimiter(rate.Every(1*time.Second), 0) // No tokens available + backoff := RateLimitedBackoffLimiter(logger, limiter, maxWait) + + min := 100 * time.Millisecond + max := 500 * time.Millisecond + attemptNum := 3 + + delay := backoff(min, max, attemptNum, nil) + assert.Equal(t, maxWait, delay) + }) +} diff --git a/pkg/client/util/util.go b/pkg/client/util/util.go index 018a2c62..cdce903e 100644 --- a/pkg/client/util/util.go +++ b/pkg/client/util/util.go @@ -46,12 +46,13 @@ func JoinRepoImage(repo, image string) string { } // Attempt to determine the OS and Arch, given a tag name. -func OSArchFromTag(tag string) (api.OS, api.Architecture) { - var ( - os api.OS - arch api.Architecture - split = strings.Split(tag, "-") - ) +func OSArchFromTag(tag string) (os api.OS, arch api.Architecture) { + split := strings.Split(tag, "-") + + // If we don't have >3 splits, then we may not have + if len(split) == 2 { + os = api.OS("linux") + } for _, s := range split { ss := strings.ToLower(s) @@ -71,3 +72,22 @@ func OSArchFromTag(tag string) (api.OS, api.Architecture) { return os, arch } + +func TagMaptoList(tags map[string]api.ImageTag) []api.ImageTag { + taglist := make([]api.ImageTag, 0, len(tags)) + for _, tag := range tags { + taglist = append(taglist, tag) + } + return taglist +} + +func BuildTags(tags map[string]api.ImageTag, tag string, current *api.ImageTag) { + // Already exists — add as child + if parent, exists := tags[tag]; exists { + parent.Children = append(parent.Children, current) + tags[tag] = parent + } else { + // First occurrence — assign as root + tags[tag] = *current + } +} diff --git a/pkg/client/util/util_test.go b/pkg/client/util/util_test.go index 2955055a..39d0bd9f 100644 --- a/pkg/client/util/util_test.go +++ b/pkg/client/util/util_test.go @@ -2,6 +2,10 @@ package util import ( "testing" + + "github.com/stretchr/testify/assert" + + "github.com/jetstack/version-checker/pkg/api" ) func TestJoinRepoImage(t *testing.T) { @@ -45,3 +49,89 @@ func TestJoinRepoImage(t *testing.T) { }) } } + +func TestOSArchFromTag(t *testing.T) { + tests := map[string]struct { + tag string + expOS api.OS + expArch api.Architecture + }{ + "empty tag should return empty OS and Arch": { + tag: "", + expOS: "", + expArch: "", + }, + "tag with only OS should return correct OS and empty Arch": { + tag: "v1.0.0-linux", + expOS: "linux", + expArch: "", + }, + "tag with only Arch should return linux OS and correct Arch": { + tag: "v1.0.0-amd64", + expOS: "linux", + expArch: "amd64", + }, + "tag with OS and Arch should return both correctly": { + tag: "v1.0.0-linux-amd64", + expOS: "linux", + expArch: "amd64", + }, + "tag with unknown OS and Arch should return empty OS and Arch": { + tag: "v1.0.0-os-unknown-arch", + expOS: "", + expArch: "", + }, + "tag with mixed case OS and Arch should return correct OS and Arch": { + tag: "v1.0.0-Linux-AMD64", + expOS: "linux", + expArch: "amd64", + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + os, arch := OSArchFromTag(test.tag) + + assert.Equal(t, os, test.expOS) + assert.Equal(t, arch, test.expArch) + }) + } +} +func TestTagMaptoList(t *testing.T) { + tests := map[string]struct { + tags map[string]api.ImageTag + expList []api.ImageTag + }{ + "empty map should return empty list": { + tags: map[string]api.ImageTag{}, + expList: []api.ImageTag{}, + }, + "single entry map should return single element list": { + tags: map[string]api.ImageTag{ + "v1.0.0": {Tag: "v1.0.0"}, + }, + expList: []api.ImageTag{ + {Tag: "v1.0.0"}, + }, + }, + "multiple entry map should return list with all elements": { + tags: map[string]api.ImageTag{ + "v1.0.0": {Tag: "v1.0.0"}, + "v1.1.0": {Tag: "v1.1.0"}, + "v2.0.0": {Tag: "v2.0.0"}, + }, + expList: []api.ImageTag{ + {Tag: "v1.0.0"}, + {Tag: "v1.1.0"}, + {Tag: "v2.0.0"}, + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + result := TagMaptoList(test.tags) + assert.ElementsMatch(t, result, test.expList) + }) + } +} diff --git a/pkg/controller/checker/checker.go b/pkg/controller/checker/checker.go index a80f81ba..387d2ad1 100644 --- a/pkg/controller/checker/checker.go +++ b/pkg/controller/checker/checker.go @@ -20,8 +20,8 @@ type Checker struct { type Result struct { CurrentVersion string LatestVersion string - IsLatest bool ImageURL string + IsLatest bool } func New(search search.Searcher) *Checker { @@ -145,11 +145,24 @@ func (c *Checker) isLatestSemver(ctx context.Context, imageURL, currentSHA strin isLatest = true } - // If using the same image version, but the SHA has been updated upstream, - // make not latest - if currentImage.Equal(latestImageV) && currentSHA != latestImage.SHA && latestImage.SHA != "" { + // If using the same image version, + // but the SHA has been updated upstream, + // mark not latest + if currentImage.Equal(latestImageV) && + !latestImage.MatchesSHA(currentSHA) { isLatest = false - latestImage.Tag = fmt.Sprintf("%s@%s", latestImage.Tag, latestImage.SHA) + if latestImage.SHA != "" { + // Add the SHA as a prefix to identify that it has been updated! + latestImage.Tag = fmt.Sprintf("%s@%s", latestImage.Tag, latestImage.SHA) + } else { + for _, child := range latestImage.Children { + // Take first child's SHA for latest image tag + if child.SHA != currentSHA { + latestImage.Tag = fmt.Sprintf("%s@%s", latestImage.Tag, child.SHA) + break + } + } + } } return latestImage, isLatest, nil @@ -162,10 +175,26 @@ func (c *Checker) isLatestSHA(ctx context.Context, imageURL, currentSHA string, return nil, err } - isLatest := latestImage.SHA == currentSHA - latestVersion := latestImage.SHA - if len(latestImage.Tag) > 0 { - latestVersion = fmt.Sprintf("%s@%s", latestImage.Tag, latestImage.SHA) + var ( + isLatest bool + latestVersion string + ) + + if latestImage.SHA != "" { + isLatest = latestImage.SHA == currentSHA + latestVersion = latestImage.SHA + } + + for _, child := range latestImage.Children { + if child.SHA == currentSHA { + isLatest = true + latestVersion = child.SHA + break + } + } + + if len(latestImage.Tag) > 0 && latestVersion != "" { + latestVersion = fmt.Sprintf("%s@%s", latestImage.Tag, latestVersion) } return &Result{ diff --git a/pkg/controller/checker/checker_test.go b/pkg/controller/checker/checker_test.go index ac5dadd6..b5d1f598 100644 --- a/pkg/controller/checker/checker_test.go +++ b/pkg/controller/checker/checker_test.go @@ -8,6 +8,9 @@ import ( "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/jetstack/version-checker/pkg/api" "github.com/jetstack/version-checker/pkg/controller/internal/fake/search" "github.com/jetstack/version-checker/pkg/version/semver" @@ -58,6 +61,54 @@ func TestContainer(t *testing.T) { IsLatest: true, }, }, + "if v0.2.0 is latest version, but sha is in a child, then latest": { + statusSHA: "localhost:5000/version-checker@sha:123", + imageURL: "localhost:5000/version-checker:v0.2.0", + opts: new(api.Options), + searchResp: &api.ImageTag{ + Tag: "v0.2.0", + SHA: "sha:abc1234", + Children: []*api.ImageTag{{SHA: "sha:123"}}, + }, + expResult: &Result{ + CurrentVersion: "v0.2.0", + LatestVersion: "v0.2.0", + ImageURL: "localhost:5000/version-checker", + IsLatest: true, + }, + }, + "if v0.2.0 is latest version, but sha is not in cache, then not latest": { + statusSHA: "localhost:5000/version-checker@sha:123", + imageURL: "localhost:5000/version-checker:v0.2.0", + opts: new(api.Options), + searchResp: &api.ImageTag{ + Tag: "v0.2.0", + SHA: "", + Children: []*api.ImageTag{{SHA: "sha:789"}}, + }, + expResult: &Result{ + CurrentVersion: "v0.2.0@sha:123", + LatestVersion: "v0.2.0@sha:789", + ImageURL: "localhost:5000/version-checker", + IsLatest: false, + }, + }, + "if v0.2.0 is latest version, but sha is not in cache, and multiple possible shas, then not latest": { + statusSHA: "localhost:5000/version-checker@123", + imageURL: "localhost:5000/version-checker:v0.2.0", + opts: new(api.Options), + searchResp: &api.ImageTag{ + Tag: "v0.2.0", + SHA: "", + Children: []*api.ImageTag{{SHA: "789"}, {SHA: "sha:987"}}, + }, + expResult: &Result{ + CurrentVersion: "v0.2.0@123", + LatestVersion: "v0.2.0@789", + ImageURL: "localhost:5000/version-checker", + IsLatest: false, + }, + }, "if v0.2.0@sha:123 is wrong sha, then not latest": { statusSHA: "localhost:5000/version-checker@sha:123", imageURL: "localhost:5000/version-checker:v0.2.0@sha:123", @@ -280,14 +331,8 @@ func TestContainer(t *testing.T) { } result, err := checker.Container(context.TODO(), logrus.NewEntry(logrus.New()), pod, container, test.opts) - if err != nil { - t.Errorf("unexpected error: %s", err) - } - - if !reflect.DeepEqual(test.expResult, result) { - t.Errorf("got unexpected result, exp=%#+v got=%#+v", - test.expResult, result) - } + require.NoError(t, err) + assert.Exactly(t, test.expResult, result) }) } } @@ -500,20 +545,21 @@ func TestIsLatestSHA(t *testing.T) { searchResp *api.ImageTag expResult *Result }{ - "if SHA not eqaual, then should be not equal": { + "if SHA not equal, then should be not equal": { imageURL: "docker.io", currentSHA: "123", searchResp: &api.ImageTag{ SHA: "456", + Tag: "foo", }, expResult: &Result{ CurrentVersion: "123", - LatestVersion: "456", + LatestVersion: "foo@456", IsLatest: false, ImageURL: "docker.io", }, }, - "if SHA eqaual, then should be equal": { + "if SHA equal, then should be equal": { imageURL: "docker.io", currentSHA: "123", searchResp: &api.ImageTag{ @@ -526,6 +572,60 @@ func TestIsLatestSHA(t *testing.T) { ImageURL: "docker.io", }, }, + "if child SHA equal, then should be equal": { + imageURL: "docker.io", + currentSHA: "123", + searchResp: &api.ImageTag{ + SHA: "456", + Tag: "foo", + Children: []*api.ImageTag{ + { + SHA: "123", + }, + }, + }, + expResult: &Result{ + CurrentVersion: "123", + LatestVersion: "foo@123", + IsLatest: true, + ImageURL: "docker.io", + }, + }, + "if child SHA equal, and parent SHA empty, then should be equal": { + imageURL: "docker.io", + currentSHA: "123", + searchResp: &api.ImageTag{ + SHA: "", + Tag: "foo", + Children: []*api.ImageTag{ + {SHA: "123"}, + {SHA: "456"}, + }, + }, + expResult: &Result{ + CurrentVersion: "123", + LatestVersion: "foo@123", + IsLatest: true, + ImageURL: "docker.io", + }, + }, + "if child SHA not equal, and parent SHA empty, then should not be equal": { + imageURL: "docker.io", + currentSHA: "123", + searchResp: &api.ImageTag{ + SHA: "", + Children: []*api.ImageTag{ + {SHA: "456"}, + {SHA: "789"}, + }, + }, + expResult: &Result{ + CurrentVersion: "123", + LatestVersion: "", + IsLatest: false, + ImageURL: "docker.io", + }, + }, } for name, test := range tests { diff --git a/pkg/controller/options/options.go b/pkg/controller/options/options.go index f5264450..20ae0798 100644 --- a/pkg/controller/options/options.go +++ b/pkg/controller/options/options.go @@ -65,12 +65,14 @@ func (b *Builder) Options(name string) (*api.Options, error) { return &opts, nil } + func (b *Builder) handleSHAOption(name string, opts *api.Options, setNonSha *bool, errs *[]string) error { if useSHA, ok := b.ans[b.index(name, api.UseSHAAnnotationKey)]; ok && useSHA == "true" { opts.UseSHA = true } return nil } + func (b *Builder) handleSHAToTagOption(name string, opts *api.Options, setNonSha *bool, errs *[]string) error { if ResolveSHAToTags, ok := b.ans[b.index(name, api.ResolveSHAToTagsKey)]; ok && ResolveSHAToTags == "true" { opts.ResolveSHAToTags = true diff --git a/pkg/controller/pod_controller.go b/pkg/controller/pod_controller.go index 60a8bd6d..1f39cd3f 100644 --- a/pkg/controller/pod_controller.go +++ b/pkg/controller/pod_controller.go @@ -12,6 +12,7 @@ import ( k8sclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" "github.com/jetstack/version-checker/pkg/client" @@ -50,7 +51,7 @@ func NewPodReconciler( versionGetter := version.New(log, imageClient, cacheTimeout) search := search.New(log, cacheTimeout, versionGetter) - c := &PodReconciler{ + return &PodReconciler{ Log: log, Client: kubeClient, Metrics: metrics, @@ -58,8 +59,6 @@ func NewPodReconciler( RequeueDuration: requeueDuration, defaultTestAll: defaultTestAll, } - - return c } // Reconcile is triggered whenever a watched object changes. @@ -79,7 +78,7 @@ func (r *PodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R return ctrl.Result{}, err } - // Perform the version check (your sync logic) + // Perform the version check if err := r.sync(ctx, pod); err != nil { log.Error(err, "Failed to process pod") // Requeue after some time in case of failure @@ -95,10 +94,22 @@ func (r *PodReconciler) SetupWithManager(mgr ctrl.Manager) error { LeaderElect := false return ctrl.NewControllerManagedBy(mgr). For(&corev1.Pod{}, builder.OnlyMetadata). - WithOptions(controller.Options{MaxConcurrentReconciles: numWorkers, NeedLeaderElection: &LeaderElect}). + WithOptions(controller.Options{ + MaxConcurrentReconciles: numWorkers, + NeedLeaderElection: &LeaderElect, + }). WithEventFilter(predicate.Funcs{ CreateFunc: func(_ event.TypedCreateEvent[k8sclient.Object]) bool { return true }, - UpdateFunc: func(_ event.TypedUpdateEvent[k8sclient.Object]) bool { return true }, + UpdateFunc: func(e event.TypedUpdateEvent[k8sclient.Object]) bool { + oldAnn := e.ObjectOld.GetAnnotations() + newAnn := e.ObjectNew.GetAnnotations() + + if !annotationsEqual(oldAnn, newAnn) { + // Remove metrics for pod, if the annotations have changed + r.Metrics.RemovePod(e.ObjectOld.GetNamespace(), e.ObjectOld.GetName()) + } + return true + }, DeleteFunc: func(e event.TypedDeleteEvent[k8sclient.Object]) bool { r.Log.Infof("Pod deleted: %s/%s", e.Object.GetNamespace(), e.Object.GetName()) r.Metrics.RemovePod(e.Object.GetNamespace(), e.Object.GetName()) @@ -107,3 +118,16 @@ func (r *PodReconciler) SetupWithManager(mgr ctrl.Manager) error { }). Complete(r) } + +// annotationsEqual compares two annotation maps for equality. +func annotationsEqual(a, b map[string]string) bool { + if len(a) != len(b) { + return false + } + for key, valA := range a { + if valB, ok := b[key]; !ok || valA != valB { + return false + } + } + return true +} diff --git a/pkg/controller/pod_controller_test.go b/pkg/controller/pod_controller_test.go index 0828f1fc..aa97a2fb 100644 --- a/pkg/controller/pod_controller_test.go +++ b/pkg/controller/pod_controller_test.go @@ -48,6 +48,7 @@ func TestNewController(t *testing.T) { assert.Equal(t, controller.Client, kubeClient) assert.NotNil(t, controller.VersionChecker) } + func TestReconcile(t *testing.T) { imageClient := &client.Client{} diff --git a/pkg/controller/search/search.go b/pkg/controller/search/search.go index e75daccd..79e4b49a 100644 --- a/pkg/controller/search/search.go +++ b/pkg/controller/search/search.go @@ -20,6 +20,9 @@ type Searcher interface { ResolveSHAToTag(ctx context.Context, imageURL string, imageSHA string) (string, error) } +// Ensure The search Struct implements a cacheHandler +var _ cache.Handler = (*Search)(nil) + // Search is the implementation for the searching and caching of image URLs. type Search struct { log *logrus.Entry @@ -40,7 +43,7 @@ func New(log *logrus.Entry, cacheTimeout time.Duration, versionGetter *version.V return s } -func (s *Search) Fetch(ctx context.Context, imageURL string, opts *api.Options) (interface{}, error) { +func (s *Search) Fetch(ctx context.Context, imageURL string, opts *api.Options) (any, error) { latestImage, err := s.versionGetter.LatestTagFromImage(ctx, imageURL, opts) if err != nil { return nil, err @@ -67,7 +70,6 @@ func (s *Search) LatestImage(ctx context.Context, imageURL string, opts *api.Opt } func (s *Search) ResolveSHAToTag(ctx context.Context, imageURL string, imageSHA string) (string, error) { - tag, err := s.versionGetter.ResolveSHAToTag(ctx, imageURL, imageSHA) if err != nil { return "", fmt.Errorf("failed to resolve sha to tag: %w", err) diff --git a/pkg/version/filters.go b/pkg/version/filters.go index f8360581..314c325d 100644 --- a/pkg/version/filters.go +++ b/pkg/version/filters.go @@ -15,6 +15,10 @@ func isSBOMAttestationOrSig(tag string) bool { // Used when filtering Tags as a SemVer func shouldSkipTag(opts *api.Options, v *semver.SemVer) bool { + if isSBOMAttestationOrSig(v.String()) { + return true + } + // Handle Regex matching if opts.RegexMatcher != nil { return !opts.RegexMatcher.MatchString(v.String()) diff --git a/pkg/version/helpers.go b/pkg/version/helpers.go new file mode 100644 index 00000000..d37f8ca5 --- /dev/null +++ b/pkg/version/helpers.go @@ -0,0 +1,55 @@ +package version + +import ( + "fmt" + + "github.com/jetstack/version-checker/pkg/api" + "github.com/jetstack/version-checker/pkg/version/semver" +) + +// latestSemver will return the latest ImageTag based on the given options +// restriction, using semver. This should not be used if UseSHA has been +// enabled. +func latestSemver(opts *api.Options, tags []api.ImageTag) (*api.ImageTag, error) { + var ( + latestImageTag *api.ImageTag + latestV *semver.SemVer + ) + + for i := range tags { + v := semver.Parse(tags[i].Tag) + + if shouldSkipTag(opts, v) { + continue + } + + if isBetterSemVer(opts, latestV, v, latestImageTag, &tags[i]) { + latestV = v + latestImageTag = &tags[i] + } + } + + if latestImageTag == nil { + return nil, fmt.Errorf("no suitable version found") + } + + return latestImageTag, nil +} + +// latestSHA will return the latest ImageTag based on image timestamps. +func latestSHA(opts *api.Options, tags []api.ImageTag) (*api.ImageTag, error) { + var latestTag *api.ImageTag + + for i := range tags { + // Filter out SBOM and Attestation/Sig's... + if shouldSkipSHA(opts, tags[i].Tag) { + continue + } + + if latestTag == nil || tags[i].Timestamp.After(latestTag.Timestamp) { + latestTag = &tags[i] + } + } + + return latestTag, nil +} diff --git a/pkg/version/semver/semver.go b/pkg/version/semver/semver.go index 0fb35d9a..ad74f98e 100644 --- a/pkg/version/semver/semver.go +++ b/pkg/version/semver/semver.go @@ -12,15 +12,15 @@ var ( // SemVer is a struct to contain a SemVer of an image tag. type SemVer struct { - // version is the version number of a tag. 'Left', or smaller index, the - // higher weight. - version [3]int64 // metadata holds the metadata, which is the string suffixed from the patch metadata string // original holds the origin string of the tag original string + // version is the version number of a tag. 'Left', or smaller index, the + // higher weight. + version [3]int64 } func Parse(tag string) *SemVer { diff --git a/pkg/version/version.go b/pkg/version/version.go index b34a846d..b0fb5ebd 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -13,9 +13,10 @@ import ( "github.com/jetstack/version-checker/pkg/cache" versionerrors "github.com/jetstack/version-checker/pkg/version/errors" - "github.com/jetstack/version-checker/pkg/version/semver" ) +var _ cache.Handler = (*Version)(nil) + type Version struct { log *logrus.Entry @@ -55,8 +56,8 @@ func (v *Version) LatestTagFromImage(ctx context.Context, imageURL string, opts } if tag == nil { - return nil, versionerrors.NewVersionErrorNotFound("%s: failed to find latest image based on SHA", - imageURL) + return nil, versionerrors.NewVersionErrorNotFound( + "%s: failed to find latest image based on SHA", imageURL) } } else { tag, err = latestSemver(opts, tags) @@ -66,7 +67,8 @@ func (v *Version) LatestTagFromImage(ctx context.Context, imageURL string, opts if tag == nil { optsBytes, _ := json.Marshal(opts) - return nil, versionerrors.NewVersionErrorNotFound("%s: no tags found with these option constraints: %s", + return nil, versionerrors.NewVersionErrorNotFound( + "%s: no tags found with these option constraints: %s", imageURL, optsBytes) } } @@ -76,7 +78,6 @@ func (v *Version) LatestTagFromImage(ctx context.Context, imageURL string, opts // ResolveSHAToTag Resolve a SHA to a tag if possible func (v *Version) ResolveSHAToTag(ctx context.Context, imageURL string, imageSHA string) (string, error) { - tagsI, err := v.imageCache.Get(ctx, imageURL, imageURL, nil) if err != nil { return "", err @@ -84,7 +85,7 @@ func (v *Version) ResolveSHAToTag(ctx context.Context, imageURL string, imageSHA tags := tagsI.([]api.ImageTag) for i := range tags { - if tags[i].SHA == imageSHA { + if tags[i].MatchesSHA(imageSHA) { return tags[i].Tag, nil } } @@ -110,55 +111,3 @@ func (v *Version) Fetch(ctx context.Context, imageURL string, _ *api.Options) (i return tags, nil } - -// latestSemver will return the latest ImageTag based on the given options -// restriction, using semver. This should not be used is UseSHA has been -// enabled. -func latestSemver(opts *api.Options, tags []api.ImageTag) (*api.ImageTag, error) { - var ( - latestImageTag *api.ImageTag - latestV *semver.SemVer - ) - - for i := range tags { - // Filter out SBOM and Attestation/Sig's - if isSBOMAttestationOrSig(tags[i].Tag) || isSBOMAttestationOrSig(tags[i].SHA) { - continue - } - - v := semver.Parse(tags[i].Tag) - - if shouldSkipTag(opts, v) { - continue - } - - if isBetterSemVer(opts, latestV, v, latestImageTag, &tags[i]) { - latestV = v - latestImageTag = &tags[i] - } - } - - if latestImageTag == nil { - return nil, fmt.Errorf("no suitable version found") - } - - return latestImageTag, nil -} - -// latestSHA will return the latest ImageTag based on image timestamps. -func latestSHA(opts *api.Options, tags []api.ImageTag) (*api.ImageTag, error) { - var latestTag *api.ImageTag - - for i := range tags { - // Filter out SBOM and Attestation/Sig's... - if shouldSkipSHA(opts, tags[i].Tag) { - continue - } - - if latestTag == nil || tags[i].Timestamp.After(latestTag.Timestamp) { - latestTag = &tags[i] - } - } - - return latestTag, nil -}