diff --git a/pkg/minikube/machine/cache_images.go b/pkg/minikube/machine/cache_images.go
index a2d757a79b15..27d0ef084df8 100644
--- a/pkg/minikube/machine/cache_images.go
+++ b/pkg/minikube/machine/cache_images.go
@@ -329,6 +329,11 @@ func removeExistingImage(r cruntime.Manager, src string, imgName string) error {
 	}
 
 	errStr := strings.ToLower(err.Error())
+	if strings.Contains(errStr, "is using its referenced image") {
+		klog.Warningf("Image %s is being used by a running container, skipping deletion, it would be replaced later", imgName)
+		return nil
+	}
+
 	if !strings.Contains(errStr, "no such image") {
 		return errors.Wrap(err, "removing image")
 	}