Merge pull request #56 from bitrise-io/skip-existing-files

Author: imrekel

.golangci.yaml

@@ -29,3 +29,4 @@ linters:
   - gci
   - nolintlint
   - cyclop
+  - execinquery # deprecated (since v1.58.0) due to: The repository of the linter has been archived by the owner.

cmd/restoreXcodeDerivedDataFiles.go

@@ -47,8 +47,12 @@ var restoreXcodeDerivedDataFilesCmd = &cobra.Command{
 		projectRoot, _ := cmd.Flags().GetString("project-root")
 		cacheKey, _ := cmd.Flags().GetString("key")
 		forceOverwrite, _ := cmd.Flags().GetBool("force-overwrite-files")
+		skipExisting, _ := cmd.Flags().GetBool("skip-existing-files")
 		maxLoggedErrors, _ := cmd.Flags().GetInt("max-logged-errors")
 
+		logger.Infof("(i) Skip existing files: %t", skipExisting)
+		logger.Infof("(i) Force overwrite existing files: %t", forceOverwrite)
+
 		tracker := xcode.NewDefaultStepTracker("restore-xcode-build-cache", os.Getenv, logger)
 		defer tracker.Wait()
 		startT := time.Now()
@@ -60,7 +64,7 @@ var restoreXcodeDerivedDataFilesCmd = &cobra.Command{
 		}
 
 		op, cmdError := restoreXcodeDerivedDataFilesCmdFn(cmd.Context(), authConfig, CacheMetadataPath, projectRoot,
-			cacheKey, logger, tracker, startT, os.Getenv, isDebugLogMode, forceOverwrite, maxLoggedErrors)
+			cacheKey, logger, tracker, startT, os.Getenv, isDebugLogMode, skipExisting, forceOverwrite, maxLoggedErrors)
 		if op != nil {
 			if cmdError != nil {
 				errStr := cmdError.Error()
@@ -92,11 +96,12 @@ func init() {
 		panic(err)
 	}
 	restoreXcodeDerivedDataFilesCmd.Flags().Bool("force-overwrite-files", false, "If set, the command will try to overwrite existing files during restoring the cache even if the permissions do not allow it")
+	restoreXcodeDerivedDataFilesCmd.Flags().Bool("skip-existing-files", false, "If set, existing files will be skipped and not be overwritten during restoring the cache")
 	restoreXcodeDerivedDataFilesCmd.Flags().Int("max-logged-errors", 150, "The maximum number of errors logged to the console during restoring the cache.")
 }
 
 func restoreXcodeDerivedDataFilesCmdFn(ctx context.Context, authConfig common.CacheAuthConfig, cacheMetadataPath, projectRoot, providedCacheKey string, logger log.Logger,
-	tracker xcode.StepAnalyticsTracker, startT time.Time, envProvider func(string) string, isDebugLogMode, forceOverwrite bool, maxLoggedDownloadErrors int) (*xa.CacheOperation, error) {
+	tracker xcode.StepAnalyticsTracker, startT time.Time, envProvider func(string) string, isDebugLogMode, skipExisting, forceOverwrite bool, maxLoggedDownloadErrors int) (*xa.CacheOperation, error) {
 	op := newCacheOperation(startT, xa.OperationTypeDownload, envProvider)
 	kvClient, err := createKVClient(ctx, op.OperationID, authConfig, envProvider, logger)
 	if err != nil {
@@ -137,7 +142,7 @@ func restoreXcodeDerivedDataFilesCmdFn(ctx context.Context, authConfig common.Ca
 	tracker.LogMetadataLoaded(metadataRestoredT.Sub(startT), string(cacheKeyType), len(metadata.ProjectFiles.Files)+len(metadata.ProjectFiles.Directories), filesUpdated, metadataSize)
 
 	logger.TInfof("Downloading DerivedData files")
-	stats, err := xcode.DownloadCacheFilesFromBuildCache(ctx, metadata.DerivedData, kvClient, logger, isDebugLogMode, forceOverwrite, maxLoggedDownloadErrors)
+	stats, err := xcode.DownloadCacheFilesFromBuildCache(ctx, metadata.DerivedData, kvClient, logger, isDebugLogMode, skipExisting, forceOverwrite, maxLoggedDownloadErrors)
 	ddDownloadedT := time.Now()
 	tracker.LogDerivedDataDownloaded(ddDownloadedT.Sub(metadataRestoredT), stats)
 	fillCacheOperationWithDownloadStats(op, stats)
@@ -170,7 +175,7 @@ func restoreXcodeDerivedDataFilesCmdFn(ctx context.Context, authConfig common.Ca
 
 	if len(metadata.XcodeCacheDir.Files) > 0 {
 		logger.TInfof("Downloading Xcode cache files")
-		if _, err := xcode.DownloadCacheFilesFromBuildCache(ctx, metadata.XcodeCacheDir, kvClient, logger, isDebugLogMode, forceOverwrite, maxLoggedDownloadErrors); err != nil {
+		if _, err := xcode.DownloadCacheFilesFromBuildCache(ctx, metadata.XcodeCacheDir, kvClient, logger, isDebugLogMode, skipExisting, forceOverwrite, maxLoggedDownloadErrors); err != nil {
 			return op, fmt.Errorf("download Xcode cache files: %w", err)
 		}
 

internal/xcode/download.go

@@ -25,7 +25,9 @@ var ErrFileExistsAndNotWritable = errors.New("file already exists and is not wri
 func DownloadFileFromBuildCache(ctx context.Context, fileName, key string, kvClient *kv.Client, logger log.Logger) error {
 	logger.Debugf("Downloading %s", fileName)
 
-	return downloadFile(ctx, kvClient, fileName, key, 0, logger, false, false)
+	_, err := downloadFile(ctx, kvClient, fileName, key, 0, logger, false, false, false)
+
+	return err
 }
 
 func DownloadStreamFromBuildCache(ctx context.Context, destination io.Writer, key string, kvClient *kv.Client, logger log.Logger) error {
@@ -35,29 +37,33 @@ func DownloadStreamFromBuildCache(ctx context.Context, destination io.Writer, ke
 }
 
 // nolint: nestif
-func downloadFile(ctx context.Context, client *kv.Client, filePath, key string, fileMode os.FileMode, logger log.Logger, isDebugLogMode, forceOverwrite bool) error {
+func downloadFile(ctx context.Context, client *kv.Client, filePath, key string, fileMode os.FileMode, logger log.Logger, isDebugLogMode, skipExisting, forceOverwrite bool) (bool, error) {
 	dir := filepath.Dir(filePath)
 	if err := os.MkdirAll(dir, os.ModePerm); err != nil {
-		return fmt.Errorf("create directory: %w", err)
+		return false, fmt.Errorf("create directory: %w", err)
 	}
 
 	if fileMode == 0 {
 		fileMode = 0666
 	}
 
 	if fileInfo, err := os.Stat(filePath); err == nil {
+		if skipExisting {
+			return true, nil
+		}
+
 		ownerWritable := (fileInfo.Mode().Perm() & 0200) != 0
 		if !ownerWritable {
 			if !forceOverwrite {
-				return ErrFileExistsAndNotWritable
+				return false, ErrFileExistsAndNotWritable
 			}
 
 			if err := os.Chmod(filePath, 0666); err != nil {
-				return fmt.Errorf("force overwrite - failed to change existing file permissions: %w", err)
+				return false, fmt.Errorf("force overwrite - failed to change existing file permissions: %w", err)
 			}
 
 			if err := os.Remove(filePath); err != nil {
-				return fmt.Errorf("force overwrite - failed to remove existing file: %w", err)
+				return false, fmt.Errorf("force overwrite - failed to remove existing file: %w", err)
 			}
 		}
 	}
@@ -68,11 +74,11 @@ func downloadFile(ctx context.Context, client *kv.Client, filePath, key string,
 			logFilePathDebugInfo(filePath, logger)
 		}
 
-		return fmt.Errorf("create %q: %w", filePath, err)
+		return false, fmt.Errorf("create %q: %w", filePath, err)
 	}
 	defer file.Close()
 
-	return downloadStream(ctx, file, client, key)
+	return false, downloadStream(ctx, file, client, key)
 }
 
 func downloadStream(ctx context.Context, destination io.Writer, client *kv.Client, key string) error {

internal/xcode/download_multi.go

@@ -29,7 +29,7 @@ type DownloadFilesStats struct {
 
 // nolint: gocognit
 func DownloadCacheFilesFromBuildCache(ctx context.Context, dd FileGroupInfo, kvClient *kv.Client, logger log.Logger,
-	isDebugLogMode, forceOverwrite bool, maxLoggedDownloadErrors int) (DownloadFilesStats, error) {
+	isDebugLogMode, skipExisting, forceOverwrite bool, maxLoggedDownloadErrors int) (DownloadFilesStats, error) {
 	var largestFileSize int64
 	for _, file := range dd.Files {
 		if file.Size > largestFileSize {
@@ -44,6 +44,7 @@ func DownloadCacheFilesFromBuildCache(ctx context.Context, dd FileGroupInfo, kvC
 	var filesMissing atomic.Int32
 	var filesFailedToDownload atomic.Int32
 	var downloadSize atomic.Int64
+	var skippedFiles atomic.Int32
 
 	var wg sync.WaitGroup
 	semaphore := make(chan struct{}, 20) // Limit parallelization
@@ -57,7 +58,13 @@ func DownloadCacheFilesFromBuildCache(ctx context.Context, dd FileGroupInfo, kvC
 
 			const retries = 3
 			err := retry.Times(retries).Wait(3 * time.Second).TryWithAbort(func(_ uint) (error, bool) {
-				err := downloadFile(ctx, kvClient, file.Path, file.Hash, file.Mode, logger, isDebugLogMode, forceOverwrite)
+				skipped, err := downloadFile(ctx, kvClient, file.Path, file.Hash, file.Mode, logger, isDebugLogMode, forceOverwrite, skipExisting)
+				if skipped {
+					skippedFiles.Add(1)
+
+					return nil, false
+				}
+
 				if errors.Is(err, ErrCacheNotFound) {
 					return err, true
 				} else if errors.Is(err, ErrFileExistsAndNotWritable) {
@@ -100,28 +107,27 @@ func DownloadCacheFilesFromBuildCache(ctx context.Context, dd FileGroupInfo, kvC
 	logger.TInfof("(i) Downloaded: %d files (%s). Missing: %d files. Failed: %d files", filesDownloaded.Load(), humanize.Bytes(uint64(downloadSize.Load())), filesMissing.Load(), filesFailedToDownload.Load())
 
 	stats := DownloadFilesStats{
-		FilesToBeDownloaded:   len(dd.Files),
+		FilesToBeDownloaded:   len(dd.Files) - int(skippedFiles.Load()),
 		FilesDownloaded:       int(filesDownloaded.Load()),
 		FilesMissing:          int(filesMissing.Load()),
 		FilesFailedToDownload: int(filesFailedToDownload.Load()),
 		DownloadSize:          downloadSize.Load(),
 		LargestFileSize:       largestFileSize,
 	}
 	logger.Debugf("Download stats:")
-	logger.Debugf("  Files to be downloaded: %d", len(dd.Files))
-	logger.Debugf("  Files downloaded: %d", int(filesDownloaded.Load()))
-	logger.Debugf("  Files missing: %d", int(filesMissing.Load()))
-	logger.Debugf("  Files failed to download: %d", int(filesFailedToDownload.Load()))
-	logger.Debugf("  Download size: %s", humanize.Bytes(uint64(downloadSize.Load())))
-	logger.Debugf("  Largest file size: %s", humanize.Bytes(uint64(largestFileSize)))
-
-	downloadErrors := int(filesFailedToDownload.Load())
-	missingFiles := int(filesMissing.Load())
-	if maxLoggedDownloadErrors < downloadErrors+missingFiles {
+	logger.Debugf("  Files to be downloaded: %d", stats.FilesToBeDownloaded)
+	logger.Debugf("  Files downloaded: %d", stats.FilesDownloaded)
+	logger.Debugf("  Files missing: %d", stats.FilesMissing)
+	logger.Debugf("  Files failed to download: %d", stats.FilesFailedToDownload)
+	logger.Debugf("  Files skipped (existing): %d", skippedFiles.Load())
+	logger.Debugf("  Download size: %s", humanize.Bytes(uint64(stats.DownloadSize)))
+	logger.Debugf("  Largest file size: %s", humanize.Bytes(uint64(stats.LargestFileSize)))
+
+	if maxLoggedDownloadErrors < stats.FilesFailedToDownload+stats.FilesMissing {
 		logger.Warnf("Too many download errors or missing files, only the first %d errors were logged", maxLoggedDownloadErrors)
 	}
 
-	if filesFailedToDownload.Load() > 0 || filesMissing.Load() > 0 {
+	if stats.FilesFailedToDownload > 0 || stats.FilesMissing > 0 {
 		return stats, fmt.Errorf("failed to download some files")
 	}