Skip to content

Commit ab10be3

Browse files
committed
feat: enable shallow cloning repos
Signed-off-by: Mmadu Manasseh <manasseh.mmadu@zapier.com>
1 parent 4676e7f commit ab10be3

File tree

9 files changed

+108
-20
lines changed

9 files changed

+108
-20
lines changed

cmd/locations.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ import (
1818

1919
func processLocations(ctx context.Context, ctr container.Container, locations []string) error {
2020
for index, location := range locations {
21-
if newLocation, err := maybeCloneGitUrl(ctx, ctr.RepoManager, ctr.Config.RepoRefreshInterval, location, ctr.VcsClient.Username()); err != nil {
21+
if newLocation, err := maybeCloneGitUrl(ctx, ctr.RepoManager, ctr.Config.RepoRefreshInterval, location, ctr.VcsClient.Username(), ctr.Config.EnableShallowClone); err != nil {
2222
return errors.Wrapf(err, "failed to clone %q", location)
2323
} else if newLocation != "" {
2424
locations[index] = newLocation
@@ -31,12 +31,12 @@ func processLocations(ctx context.Context, ctr container.Container, locations []
3131
}
3232

3333
type cloner interface {
34-
Clone(ctx context.Context, cloneUrl, branchName string) (*git.Repo, error)
34+
Clone(ctx context.Context, cloneUrl, branchName string, shallow bool) (*git.Repo, error)
3535
}
3636

3737
var ErrCannotUseQueryWithFilePath = errors.New("relative and absolute file paths cannot have query parameters")
3838

39-
func maybeCloneGitUrl(ctx context.Context, repoManager cloner, repoRefreshDuration time.Duration, location, vcsUsername string) (string, error) {
39+
func maybeCloneGitUrl(ctx context.Context, repoManager cloner, repoRefreshDuration time.Duration, location, vcsUsername string, shallow bool) (string, error) {
4040
result := strings.SplitN(location, "?", 2)
4141
if !isGitURL(result[0]) {
4242
if len(result) > 1 {
@@ -51,7 +51,7 @@ func maybeCloneGitUrl(ctx context.Context, repoManager cloner, repoRefreshDurati
5151
}
5252
cloneUrl := repoUrl.CloneURL(vcsUsername)
5353

54-
repo, err := repoManager.Clone(ctx, cloneUrl, query.Get("branch"))
54+
repo, err := repoManager.Clone(ctx, cloneUrl, query.Get("branch"), shallow)
5555
if err != nil {
5656
return "", errors.Wrap(err, "failed to clone")
5757
}

cmd/locations_test.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ type fakeCloner struct {
1919
err error
2020
}
2121

22-
func (f *fakeCloner) Clone(_ context.Context, cloneUrl, branchName string) (*git.Repo, error) {
22+
func (f *fakeCloner) Clone(_ context.Context, cloneUrl, branchName string, shallow bool) (*git.Repo, error) {
2323
f.cloneUrl = cloneUrl
2424
f.branchName = branchName
2525
return f.result, f.err
@@ -43,7 +43,7 @@ func TestMaybeCloneGitUrl_NonGitUrl(t *testing.T) {
4343
tc := tc
4444
t.Run(tc.name, func(t *testing.T) {
4545
fc := &fakeCloner{result: nil, err: nil}
46-
actual, err := maybeCloneGitUrl(ctx, fc, time.Duration(0), tc.input, testUsername)
46+
actual, err := maybeCloneGitUrl(ctx, fc, time.Duration(0), tc.input, testUsername, false)
4747
require.NoError(t, err)
4848
assert.Equal(t, "", fc.branchName)
4949
assert.Equal(t, "", fc.cloneUrl)
@@ -137,7 +137,7 @@ func TestMaybeCloneGitUrl_HappyPath(t *testing.T) {
137137
tc := tc
138138
t.Run(tc.name, func(t *testing.T) {
139139
fc := &fakeCloner{result: &git.Repo{Directory: testRoot}, err: nil}
140-
actual, err := maybeCloneGitUrl(ctx, fc, time.Duration(0), tc.input, testUsername)
140+
actual, err := maybeCloneGitUrl(ctx, fc, time.Duration(0), tc.input, testUsername, false)
141141
require.NoError(t, err)
142142
assert.Equal(t, tc.expected.branch, fc.branchName)
143143
assert.Equal(t, tc.expected.cloneUrl, fc.cloneUrl)
@@ -165,7 +165,7 @@ func TestMaybeCloneGitUrl_URLError(t *testing.T) {
165165
tc := tc
166166
t.Run(tc.name, func(t *testing.T) {
167167
fc := &fakeCloner{result: &git.Repo{Directory: testRoot}, err: nil}
168-
result, err := maybeCloneGitUrl(ctx, fc, time.Duration(0), tc.input, testUsername)
168+
result, err := maybeCloneGitUrl(ctx, fc, time.Duration(0), tc.input, testUsername, false)
169169
require.ErrorContains(t, err, tc.expected)
170170
require.Equal(t, "", result)
171171
})
@@ -193,7 +193,7 @@ func TestMaybeCloneGitUrl_CloneError(t *testing.T) {
193193
defer cancel()
194194

195195
fc := &fakeCloner{result: &git.Repo{Directory: testRoot}, err: tc.cloneError}
196-
result, err := maybeCloneGitUrl(ctx, fc, time.Duration(0), tc.input, testUsername)
196+
result, err := maybeCloneGitUrl(ctx, fc, time.Duration(0), tc.input, testUsername, false)
197197
require.ErrorContains(t, err, tc.expected)
198198
require.Equal(t, "", result)
199199
})

cmd/root.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,9 @@ func init() {
119119
newStringOpts().
120120
withDefault("kubechecks again"))
121121
stringSliceFlag(flags, "additional-apps-namespaces", "Additional namespaces other than the ArgoCDNamespace to monitor for applications.")
122+
boolFlag(flags, "enable-shallow-clone", "Enable shallow cloning for all git repos.",
123+
newBoolOpts().
124+
withDefault(false))
122125

123126
panicIfError(viper.BindPFlags(flags))
124127
setupLogOutput()

docs/usage.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ The full list of supported environment variables is described below:
4949
|`KUBECHECKS_ENABLE_HOOKS_RENDERER`|Render hooks.|`true`|
5050
|`KUBECHECKS_ENABLE_KUBECONFORM`|Enable kubeconform checks.|`true`|
5151
|`KUBECHECKS_ENABLE_PREUPGRADE`|Enable preupgrade checks.|`true`|
52+
|`KUBECHECKS_ENABLE_SHALLOW_CLONE`|Enable shallow cloning for all git repos.|`false`|
5253
|`KUBECHECKS_ENSURE_WEBHOOKS`|Ensure that webhooks are created in repositories referenced by argo.|`false`|
5354
|`KUBECHECKS_FALLBACK_K8S_VERSION`|Fallback target Kubernetes version for schema / upgrade checks.|`1.23.0`|
5455
|`KUBECHECKS_GITHUB_APP_ID`|Github App ID.|`0`|

localdev/kubechecks/values.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ deployment:
3131
reloader.stakater.com/auto: "true"
3232

3333
image:
34-
pullPolicy: Never
34+
pullPolicy: IfNotPresent
3535
name: "kubechecks"
3636
tag: ""
3737

pkg/config/config.go

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -38,12 +38,13 @@ type ServerConfig struct {
3838
OtelCollectorPort string `mapstructure:"otel-collector-port"`
3939

4040
// vcs
41-
VcsUsername string `mapstructure:"vcs-username"`
42-
VcsEmail string `mapstructure:"vcs-email"`
43-
VcsBaseUrl string `mapstructure:"vcs-base-url"`
44-
VcsUploadUrl string `mapstructure:"vcs-upload-url"` // github enterprise upload URL
45-
VcsToken string `mapstructure:"vcs-token"`
46-
VcsType string `mapstructure:"vcs-type"`
41+
VcsUsername string `mapstructure:"vcs-username"`
42+
VcsEmail string `mapstructure:"vcs-email"`
43+
VcsBaseUrl string `mapstructure:"vcs-base-url"`
44+
VcsUploadUrl string `mapstructure:"vcs-upload-url"` // github enterprise upload URL
45+
VcsToken string `mapstructure:"vcs-token"`
46+
VcsType string `mapstructure:"vcs-type"`
47+
EnableShallowClone bool `mapstructure:"enable-shallow-clone"`
4748

4849
//github
4950
GithubPrivateKey string `mapstructure:"github-private-key"`

pkg/events/check.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ type CheckEvent struct {
5555
}
5656

5757
type repoManager interface {
58-
Clone(ctx context.Context, cloneURL, branchName string) (*git.Repo, error)
58+
Clone(ctx context.Context, cloneURL, branchName string, shallow bool) (*git.Repo, error)
5959
}
6060

6161
func generateMatcher(ce *CheckEvent, repo *git.Repo) error {
@@ -192,7 +192,7 @@ func (ce *CheckEvent) getRepo(ctx context.Context, cloneURL, branchName string)
192192
return repo, nil
193193
}
194194

195-
repo, err = ce.repoManager.Clone(ctx, cloneURL, branchName)
195+
repo, err = ce.repoManager.Clone(ctx, cloneURL, branchName, ce.ctr.Config.EnableShallowClone)
196196
if err != nil {
197197
return nil, errors.Wrap(err, "failed to clone repo")
198198
}

pkg/git/manager.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,11 @@ func NewRepoManager(cfg config.ServerConfig) *RepoManager {
2222
return &RepoManager{cfg: cfg}
2323
}
2424

25-
func (rm *RepoManager) Clone(ctx context.Context, cloneUrl, branchName string) (*Repo, error) {
25+
func (rm *RepoManager) Clone(ctx context.Context, cloneUrl, branchName string, shallow bool) (*Repo, error) {
2626
repo := New(rm.cfg, cloneUrl, branchName)
27+
if shallow {
28+
repo.Shallow = true
29+
}
2730

2831
if err := repo.Clone(ctx); err != nil {
2932
return nil, errors.Wrap(err, "failed to clone repository")

pkg/git/repo.go

Lines changed: 81 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ type Repo struct {
2828
BranchName string
2929
Config config.ServerConfig
3030
CloneURL string
31+
Shallow bool
3132

3233
// exposed state
3334
Directory string
@@ -46,6 +47,10 @@ func New(cfg config.ServerConfig, cloneUrl, branchName string) *Repo {
4647
}
4748

4849
func (r *Repo) Clone(ctx context.Context) error {
50+
if r.Shallow {
51+
return r.ShallowClone(ctx)
52+
}
53+
4954
var err error
5055

5156
r.Directory, err = os.MkdirTemp("/tmp", "kubechecks-repo-")
@@ -85,6 +90,61 @@ func (r *Repo) Clone(ctx context.Context) error {
8590
return nil
8691
}
8792

93+
func (r *Repo) ShallowClone(ctx context.Context) error {
94+
var err error
95+
96+
r.Directory, err = os.MkdirTemp("/tmp", "kubechecks-repo-")
97+
if err != nil {
98+
return errors.Wrap(err, "failed to make temp dir")
99+
}
100+
101+
log.Info().
102+
Str("temp-dir", r.Directory).
103+
Str("clone-url", r.CloneURL).
104+
Str("branch", r.BranchName).
105+
Msg("cloning git repo")
106+
107+
// Attempt to locally clone the repo based on the provided information stored within
108+
_, span := tracer.Start(ctx, "ShallowCloneRepo")
109+
defer span.End()
110+
111+
args := []string{"clone", r.CloneURL, r.Directory, "--depth", "1"}
112+
cmd := r.execGitCommand(args...)
113+
out, err := cmd.CombinedOutput()
114+
if err != nil {
115+
log.Error().Err(err).Msgf("unable to clone repository, %s", out)
116+
return err
117+
}
118+
119+
if r.BranchName != "HEAD" {
120+
// Fetch SHA
121+
args = []string{"fetch", "origin", r.BranchName, "--depth", "1"}
122+
cmd = r.execGitCommand(args...)
123+
out, err = cmd.CombinedOutput()
124+
if err != nil {
125+
log.Error().Err(err).Msgf("unable to fetch %s repository, %s", r.BranchName, out)
126+
return err
127+
}
128+
// Checkout SHA
129+
args = []string{"checkout", r.BranchName}
130+
cmd = r.execGitCommand(args...)
131+
out, err = cmd.CombinedOutput()
132+
if err != nil {
133+
log.Error().Err(err).Msgf("unable to checkout branch %s repository, %s", r.BranchName, out)
134+
return err
135+
}
136+
}
137+
138+
if log.Trace().Enabled() {
139+
if err = filepath.WalkDir(r.Directory, printFile); err != nil {
140+
log.Warn().Err(err).Msg("failed to walk directory")
141+
}
142+
}
143+
144+
log.Info().Msg("repo has been cloned")
145+
return nil
146+
}
147+
88148
func printFile(s string, d fs.DirEntry, err error) error {
89149
if err != nil {
90150
return err
@@ -118,8 +178,23 @@ func (r *Repo) MergeIntoTarget(ctx context.Context, ref string) error {
118178
attribute.String("sha", ref),
119179
))
120180
defer span.End()
181+
merge_command := []string{"merge", ref}
182+
// For shallow clones, we need to pull the ref into the repo
183+
if r.Shallow {
184+
ref = strings.TrimPrefix(ref, "origin/")
185+
cmd := r.execGitCommand("fetch", "origin", fmt.Sprintf("%s:%s", ref, ref), "--depth", "1")
186+
out, err := cmd.CombinedOutput()
187+
if err != nil {
188+
telemetry.SetError(span, err, "fetch origin ref")
189+
log.Error().Err(err).Msgf("unable to fetch ref %s, %s", ref, out)
190+
return err
191+
}
192+
// When merging shallow clones, we need to allow unrelated histories
193+
// and use the "theirs" strategy to avoid conflicts
194+
merge_command = []string{"merge", ref, "--allow-unrelated-histories", "-X", "theirs"}
195+
}
121196

122-
cmd := r.execGitCommand("merge", ref)
197+
cmd := r.execGitCommand(merge_command...)
123198
out, err := cmd.CombinedOutput()
124199
if err != nil {
125200
telemetry.SetError(span, err, "merge commit into branch")
@@ -130,7 +205,12 @@ func (r *Repo) MergeIntoTarget(ctx context.Context, ref string) error {
130205
return nil
131206
}
132207

208+
// Since we're shallow cloning, to update we need to wipe the directory and re-clone
133209
func (r *Repo) Update(ctx context.Context) error {
210+
if r.Shallow {
211+
r.Wipe()
212+
return r.Clone(ctx)
213+
}
134214
cmd := r.execGitCommand("pull")
135215
cmd.Stdout = os.Stdout
136216
cmd.Stderr = os.Stdout

0 commit comments

Comments
 (0)