Skip to content

Commit 4b51cd6

Browse files
imrekelimrekel
authored
Merge pull request #50 from bitrise-io/existing-files
2 parents ec2bde6 + 30656de commit 4b51cd6

File tree

3 files changed

+72
-13
lines changed

3 files changed

+72
-13
lines changed

‎cmd/restoreXcodeDerivedDataFiles.go

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,12 @@ import (
66
"fmt"
77
"os"
88
"os/user"
9+
"sort"
910
"strings"
1011
"time"
1112

13+
"github.com/dustin/go-humanize"
14+
1215
xa "github.com/bitrise-io/bitrise-build-cache-cli/internal/analytics"
1316
"github.com/bitrise-io/bitrise-build-cache-cli/internal/build_cache/kv"
1417
"github.com/bitrise-io/bitrise-build-cache-cli/internal/config/common"
@@ -43,6 +46,8 @@ var restoreXcodeDerivedDataFilesCmd = &cobra.Command{
4346
logger.Infof("(i) Checking parameters")
4447
projectRoot, _ := cmd.Flags().GetString("project-root")
4548
cacheKey, _ := cmd.Flags().GetString("key")
49+
forceOverwrite, _ := cmd.Flags().GetBool("force-overwrite-files")
50+
maxLoggedErrors, _ := cmd.Flags().GetInt("max-logged-errors")
4651

4752
tracker := xcode.NewDefaultStepTracker("restore-xcode-build-cache", os.Getenv, logger)
4853
defer tracker.Wait()
@@ -54,7 +59,8 @@ var restoreXcodeDerivedDataFilesCmd = &cobra.Command{
5459
return fmt.Errorf("read auth config from environments: %w", err)
5560
}
5661

57-
op, cmdError := restoreXcodeDerivedDataFilesCmdFn(cmd.Context(), authConfig, CacheMetadataPath, projectRoot, cacheKey, logger, tracker, startT, os.Getenv, isDebugLogMode)
62+
op, cmdError := restoreXcodeDerivedDataFilesCmdFn(cmd.Context(), authConfig, CacheMetadataPath, projectRoot,
63+
cacheKey, logger, tracker, startT, os.Getenv, isDebugLogMode, forceOverwrite, maxLoggedErrors)
5864
if op != nil {
5965
if cmdError != nil {
6066
errStr := cmdError.Error()
@@ -85,10 +91,12 @@ func init() {
8591
if err := restoreXcodeDerivedDataFilesCmd.MarkFlagRequired("project-root"); err != nil {
8692
panic(err)
8793
}
94+
restoreXcodeDerivedDataFilesCmd.Flags().Bool("force-overwrite-files", false, "If set, the command will try to overwrite existing files during restoring the cache even if the permissions do not allow it")
95+
restoreXcodeDerivedDataFilesCmd.Flags().Int("max-logged-errors", 100, "The maximum number of errors logged to the console during restoring the cache.")
8896
}
8997

9098
func restoreXcodeDerivedDataFilesCmdFn(ctx context.Context, authConfig common.CacheAuthConfig, cacheMetadataPath, projectRoot, providedCacheKey string, logger log.Logger,
91-
tracker xcode.StepAnalyticsTracker, startT time.Time, envProvider func(string) string, isDebugLogMode bool) (*xa.CacheOperation, error) {
99+
tracker xcode.StepAnalyticsTracker, startT time.Time, envProvider func(string) string, isDebugLogMode, forceOverwrite bool, maxLoggedDownloadErrors int) (*xa.CacheOperation, error) {
92100
kvClient, err := createKVClient(ctx, authConfig, envProvider, logger)
93101
if err != nil {
94102
return nil, fmt.Errorf("create kv client: %w", err)
@@ -110,7 +118,7 @@ func restoreXcodeDerivedDataFilesCmdFn(ctx context.Context, authConfig common.Ca
110118
return op, fmt.Errorf("load metadata: %w", err)
111119
}
112120

113-
logCacheMetadata(metadata, logger)
121+
logCacheMetadata(metadata, logger, isDebugLogMode)
114122

115123
logger.TInfof("Restoring metadata of input files")
116124
var filesUpdated int
@@ -127,7 +135,7 @@ func restoreXcodeDerivedDataFilesCmdFn(ctx context.Context, authConfig common.Ca
127135
tracker.LogMetadataLoaded(metadataRestoredT.Sub(startT), string(cacheKeyType), len(metadata.ProjectFiles.Files)+len(metadata.ProjectFiles.Directories), filesUpdated, metadataSize)
128136

129137
logger.TInfof("Downloading DerivedData files")
130-
stats, err := xcode.DownloadCacheFilesFromBuildCache(ctx, metadata.DerivedData, kvClient, logger, isDebugLogMode)
138+
stats, err := xcode.DownloadCacheFilesFromBuildCache(ctx, metadata.DerivedData, kvClient, logger, isDebugLogMode, forceOverwrite, maxLoggedDownloadErrors)
131139
ddDownloadedT := time.Now()
132140
tracker.LogDerivedDataDownloaded(ddDownloadedT.Sub(metadataRestoredT), stats)
133141
fillCacheOperationWithDownloadStats(op, stats)
@@ -146,7 +154,7 @@ func restoreXcodeDerivedDataFilesCmdFn(ctx context.Context, authConfig common.Ca
146154

147155
if len(metadata.XcodeCacheDir.Files) > 0 {
148156
logger.TInfof("Downloading Xcode cache files")
149-
if _, err := xcode.DownloadCacheFilesFromBuildCache(ctx, metadata.XcodeCacheDir, kvClient, logger, isDebugLogMode); err != nil {
157+
if _, err := xcode.DownloadCacheFilesFromBuildCache(ctx, metadata.XcodeCacheDir, kvClient, logger, isDebugLogMode, forceOverwrite, maxLoggedDownloadErrors); err != nil {
150158
return op, fmt.Errorf("download Xcode cache files: %w", err)
151159
}
152160

@@ -211,7 +219,7 @@ func downloadMetadata(ctx context.Context, cacheMetadataPath, providedCacheKey s
211219
return cacheKeyType, cacheKey, nil
212220
}
213221

214-
func logCacheMetadata(md *xcode.Metadata, logger log.Logger) {
222+
func logCacheMetadata(md *xcode.Metadata, logger log.Logger, isDebugLogMode bool) {
215223
logger.Infof("Cache metadata:")
216224
logger.Infof(" Cache key: %s", md.CacheKey)
217225
createdAt := ""
@@ -228,6 +236,24 @@ func logCacheMetadata(md *xcode.Metadata, logger log.Logger) {
228236
logger.Infof(" Xcode cache files: %d", len(md.XcodeCacheDir.Files))
229237
logger.Infof(" Build Cache CLI version: %s", md.BuildCacheCLIVersion)
230238
logger.Infof(" Metadata version: %d", md.MetadataVersion)
239+
240+
if isDebugLogMode {
241+
sortedDDFiles := make([]*xcode.FileInfo, len(md.DerivedData.Files))
242+
copy(sortedDDFiles, md.DerivedData.Files)
243+
244+
sort.Slice(sortedDDFiles, func(i, j int) bool {
245+
return sortedDDFiles[i].Size > sortedDDFiles[j].Size
246+
})
247+
248+
if len(sortedDDFiles) > 10 {
249+
sortedDDFiles = sortedDDFiles[:10]
250+
}
251+
252+
logger.Debugf(" Largest files:")
253+
for i, file := range sortedDDFiles {
254+
logger.Debugf(" %d. %s (%s)", i+1, file.Path, humanize.Bytes(uint64(file.Size)))
255+
}
256+
}
231257
}
232258

233259
func logCurrentUserInfo(logger log.Logger) {

‎internal/xcode/download.go

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,13 @@ import (
1919
// ErrCacheNotFound ...
2020
var ErrCacheNotFound = errors.New("no cache archive found for the provided keys")
2121

22+
// ErrFileExistsAndNotWritable ...
23+
var ErrFileExistsAndNotWritable = errors.New("file already exists and is not writable")
24+
2225
func DownloadFileFromBuildCache(ctx context.Context, fileName, key string, kvClient *kv.Client, logger log.Logger) error {
2326
logger.Debugf("Downloading %s", fileName)
2427

25-
return downloadFile(ctx, kvClient, fileName, key, 0, logger, false)
28+
return downloadFile(ctx, kvClient, fileName, key, 0, logger, false, false)
2629
}
2730

2831
func DownloadStreamFromBuildCache(ctx context.Context, destination io.Writer, key string, kvClient *kv.Client, logger log.Logger) error {
@@ -31,7 +34,8 @@ func DownloadStreamFromBuildCache(ctx context.Context, destination io.Writer, ke
3134
return downloadStream(ctx, destination, kvClient, key)
3235
}
3336

34-
func downloadFile(ctx context.Context, client *kv.Client, filePath, key string, fileMode os.FileMode, logger log.Logger, isDebugLogMode bool) error {
37+
// nolint: nestif
38+
func downloadFile(ctx context.Context, client *kv.Client, filePath, key string, fileMode os.FileMode, logger log.Logger, isDebugLogMode, forceOverwrite bool) error {
3539
dir := filepath.Dir(filePath)
3640
if err := os.MkdirAll(dir, os.ModePerm); err != nil {
3741
return fmt.Errorf("create directory: %w", err)
@@ -40,6 +44,24 @@ func downloadFile(ctx context.Context, client *kv.Client, filePath, key string,
4044
if fileMode == 0 {
4145
fileMode = 0666
4246
}
47+
48+
if fileInfo, err := os.Stat(filePath); err == nil {
49+
ownerWritable := (fileInfo.Mode().Perm() & 0200) != 0
50+
if !ownerWritable {
51+
if !forceOverwrite {
52+
return ErrFileExistsAndNotWritable
53+
}
54+
55+
if err := os.Chmod(filePath, 0666); err != nil {
56+
return fmt.Errorf("force overwrite - failed to change existing file permissions: %w", err)
57+
}
58+
59+
if err := os.Remove(filePath); err != nil {
60+
return fmt.Errorf("force overwrite - failed to remove existing file: %w", err)
61+
}
62+
}
63+
}
64+
4365
file, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fileMode)
4466
if err != nil {
4567
if isDebugLogMode {

‎internal/xcode/download_multi.go

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ type DownloadFilesStats struct {
2727
LargestFileSize int64
2828
}
2929

30-
func DownloadCacheFilesFromBuildCache(ctx context.Context, dd FileGroupInfo, kvClient *kv.Client, logger log.Logger, isDebugLogMode bool) (DownloadFilesStats, error) {
30+
func DownloadCacheFilesFromBuildCache(ctx context.Context, dd FileGroupInfo, kvClient *kv.Client, logger log.Logger,
31+
isDebugLogMode, forceOverwrite bool, maxLoggedDownloadErrors int) (DownloadFilesStats, error) {
3132
var largestFileSize int64
3233
for _, file := range dd.Files {
3334
if file.Size > largestFileSize {
@@ -55,10 +56,13 @@ func DownloadCacheFilesFromBuildCache(ctx context.Context, dd FileGroupInfo, kvC
5556

5657
const retries = 3
5758
err := retry.Times(retries).Wait(3 * time.Second).TryWithAbort(func(_ uint) (error, bool) {
58-
err := downloadFile(ctx, kvClient, file.Path, file.Hash, file.Mode, logger, isDebugLogMode)
59+
err := downloadFile(ctx, kvClient, file.Path, file.Hash, file.Mode, logger, isDebugLogMode, forceOverwrite)
5960
if errors.Is(err, ErrCacheNotFound) {
6061
return err, true
61-
} else if err != nil {
62+
} else if errors.Is(err, ErrFileExistsAndNotWritable) {
63+
return err, true
64+
}
65+
if err != nil {
6266
return fmt.Errorf("download file: %w", err), false
6367
}
6468

@@ -75,7 +79,9 @@ func DownloadCacheFilesFromBuildCache(ctx context.Context, dd FileGroupInfo, kvC
7579

7680
filesMissing.Add(1)
7781
case err != nil:
78-
logger.Errorf("Failed to download file %s with error: %v", file.Path, err)
82+
if filesFailedToDownload.Load() < int32(maxLoggedDownloadErrors) {
83+
logger.Errorf("Failed to download file %s with error: %v", file.Path, err)
84+
}
7985

8086
filesFailedToDownload.Add(1)
8187
default:
@@ -87,7 +93,7 @@ func DownloadCacheFilesFromBuildCache(ctx context.Context, dd FileGroupInfo, kvC
8793

8894
wg.Wait()
8995

90-
logger.TInfof("(i) Downloaded: %d files (%s). Missing: %d files", filesDownloaded.Load(), humanize.Bytes(uint64(downloadSize.Load())), filesMissing.Load())
96+
logger.TInfof("(i) Downloaded: %d files (%s). Missing: %d files. Failed: %d files", filesDownloaded.Load(), humanize.Bytes(uint64(downloadSize.Load())), filesMissing.Load(), filesFailedToDownload.Load())
9197

9298
stats := DownloadFilesStats{
9399
FilesToBeDownloaded: len(dd.Files),
@@ -105,6 +111,11 @@ func DownloadCacheFilesFromBuildCache(ctx context.Context, dd FileGroupInfo, kvC
105111
logger.Debugf(" Download size: %s", humanize.Bytes(uint64(downloadSize.Load())))
106112
logger.Debugf(" Largest file size: %s", humanize.Bytes(uint64(largestFileSize)))
107113

114+
downloadErrors := int(filesFailedToDownload.Load())
115+
if maxLoggedDownloadErrors < downloadErrors {
116+
logger.Warnf("Too many download errors (%d), only the first %d errors are logged", downloadErrors, maxLoggedDownloadErrors)
117+
}
118+
108119
if filesFailedToDownload.Load() > 0 || filesMissing.Load() > 0 {
109120
return stats, fmt.Errorf("failed to download some files")
110121
}

0 commit comments

Comments
 (0)