From ba76419dd311812b4c6a18f614e05ac1ffec5c1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xa9aX=20=E3=83=84?= Date: Sun, 25 Jul 2021 19:48:55 +0530 Subject: [PATCH 01/13] Fix deepcopy issue --- avalanche/evaluation/metrics/checkpoint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/avalanche/evaluation/metrics/checkpoint.py b/avalanche/evaluation/metrics/checkpoint.py index 82007a5c4..0a9c32cde 100644 --- a/avalanche/evaluation/metrics/checkpoint.py +++ b/avalanche/evaluation/metrics/checkpoint.py @@ -80,7 +80,7 @@ def _package_result(self, strategy) -> 'MetricResult': self.get_global_counter())] def after_eval_exp(self, strategy: 'BaseStrategy') -> 'MetricResult': - model_params = copy.deepcopy(strategy.model.parameters()) + model_params = copy.deepcopy(list(trategy.model.parameters())) self.update(model_params) def __str__(self): From d4577625347ce55813856bcd69bc3939f53bbcfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xa9aX=20=E3=83=84?= Date: Sun, 25 Jul 2021 19:52:11 +0530 Subject: [PATCH 02/13] minor fix --- avalanche/evaluation/metrics/checkpoint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/avalanche/evaluation/metrics/checkpoint.py b/avalanche/evaluation/metrics/checkpoint.py index 0a9c32cde..bef49acae 100644 --- a/avalanche/evaluation/metrics/checkpoint.py +++ b/avalanche/evaluation/metrics/checkpoint.py @@ -80,7 +80,7 @@ def _package_result(self, strategy) -> 'MetricResult': self.get_global_counter())] def after_eval_exp(self, strategy: 'BaseStrategy') -> 'MetricResult': - model_params = copy.deepcopy(list(trategy.model.parameters())) + model_params = copy.deepcopy(list(strategy.model.parameters())) self.update(model_params) def __str__(self): From e17ca755799b28253415d4247f1545f72352110c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xa9aX=20=E3=83=84?= Date: Sun, 25 Jul 2021 21:22:53 +0530 Subject: [PATCH 03/13] minor update --- examples/wandb_logger.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/examples/wandb_logger.py b/examples/wandb_logger.py index 2f7271eb4..478f94de4 100644 --- a/examples/wandb_logger.py +++ b/examples/wandb_logger.py @@ -21,6 +21,8 @@ from os.path import expanduser import argparse + +from click import Path import torch from torch.nn import CrossEntropyLoss from torch.optim import SGD @@ -74,6 +76,8 @@ def main(args): interactive_logger = InteractiveLogger() wandb_logger = WandBLogger(project_name=args.project, run_name=args.run, + log_artifacts=args.artifacts, + path= args.path if args.path else None, config=args) eval_plugin = EvaluationPlugin( @@ -127,8 +131,12 @@ def main(args): parser = argparse.ArgumentParser() parser.add_argument('--cuda', type=int, default=0, help='Select zero-indexed cuda device. -1 to use CPU.') - parser.add_argument('--run', type=str, help='Provide a run name for WandB') parser.add_argument('--project', type=str, help='Define the name of the WandB project') + parser.add_argument('--run', type=str, help='Provide a run name for WandB') + parser.add_argument('--artifacts', type=bool, default=False, + help='Log Model Checkpoints as W&B Artifacts') + parser.add_argument('--path', type=str, default=None, + help='Local path to save the model checkpoints') args = parser.parse_args() main(args) From 07bcd8500906904ac88d547fe977d8d9505611ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xa9aX=20=E3=83=84?= Date: Sun, 25 Jul 2021 23:35:59 +0530 Subject: [PATCH 04/13] Fix Pep8 issues --- examples/wandb_logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/wandb_logger.py b/examples/wandb_logger.py index 478f94de4..fb50cfac4 100644 --- a/examples/wandb_logger.py +++ b/examples/wandb_logger.py @@ -77,7 +77,7 @@ def main(args): interactive_logger = InteractiveLogger() wandb_logger = WandBLogger(project_name=args.project, run_name=args.run, log_artifacts=args.artifacts, - path= args.path if args.path else None, + path=args.path if args.path else None, config=args) eval_plugin = EvaluationPlugin( From b3d8745e12d2a6943ab10677abb046db9b9078cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xa9aX=20=E3=83=84?= Date: Thu, 5 Aug 2021 13:08:04 +0530 Subject: [PATCH 05/13] Metadata for Artifacts --- avalanche/logging/wandb_logger.py | 13 +++++++++++-- examples/wandb_logger.py | 1 - 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/avalanche/logging/wandb_logger.py b/avalanche/logging/wandb_logger.py index 550b7434a..b7d4e0545 100644 --- a/avalanche/logging/wandb_logger.py +++ b/avalanche/logging/wandb_logger.py @@ -27,6 +27,7 @@ from avalanche.evaluation.metric_results import AlternativeValues, \ MetricValue, TensorImage +from avalanche.evaluation.metrics_utils import phase_and_task from avalanche.logging import StrategyLogger @@ -84,7 +85,8 @@ def __init__(self, project_name: str = "Avalanche", def import_wandb(self): try: import wandb - except ImportError: + assert hasattr(wandb, '__version__') + except (ImportError, AssertionError): raise ImportError( 'Please run "pip install wandb" to install wandb') self.wandb = wandb @@ -150,7 +152,14 @@ def log_metric(self, metric_value: MetricValue, callback: str): if isinstance(value, Tensor): torch.save(value, dir_name) name = os.path.splittext(self.checkpoint) - artifact = self.wandb.Artifact(name, type='model') + metadata = {'experience': + self.strategy.experience.current_experience, + **({'task_id': + phase_and_task(self.strategy)[1]} + if phase_and_task(self.strategy)[1] + else {})} + artifact = self.wandb.Artifact(name, type='model', + metadata=metadata) artifact.add_file(dir_name, name=artifact_name) self.wandb.run.log_artifact(artifact) if self.uri is not None: diff --git a/examples/wandb_logger.py b/examples/wandb_logger.py index fb50cfac4..d3eaa0b06 100644 --- a/examples/wandb_logger.py +++ b/examples/wandb_logger.py @@ -22,7 +22,6 @@ import argparse -from click import Path import torch from torch.nn import CrossEntropyLoss from torch.optim import SGD From 4fd5e796b1e116f2a2dd4bb280ae2d83d6ffe622 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xa9aX=20=E3=83=84?= Date: Fri, 6 Aug 2021 06:22:28 +0530 Subject: [PATCH 06/13] Simple edits --- examples/wandb_logger.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/wandb_logger.py b/examples/wandb_logger.py index d3eaa0b06..50839113d 100644 --- a/examples/wandb_logger.py +++ b/examples/wandb_logger.py @@ -134,6 +134,7 @@ def main(args): help='Define the name of the WandB project') parser.add_argument('--run', type=str, help='Provide a run name for WandB') parser.add_argument('--artifacts', type=bool, default=False, + action="store_true", help='Log Model Checkpoints as W&B Artifacts') parser.add_argument('--path', type=str, default=None, help='Local path to save the model checkpoints') From 16856a54eb9ef3a9ed011ffa26e5350015a68090 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xa9aX=20=E3=83=84?= Date: Sat, 25 Sep 2021 10:44:26 +0530 Subject: [PATCH 07/13] Fix import error --- avalanche/logging/wandb_logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/avalanche/logging/wandb_logger.py b/avalanche/logging/wandb_logger.py index aab98f348..33869fc1a 100644 --- a/avalanche/logging/wandb_logger.py +++ b/avalanche/logging/wandb_logger.py @@ -27,7 +27,7 @@ from avalanche.evaluation.metric_results import AlternativeValues, \ MetricValue, TensorImage -from avalanche.evaluation.metrics_utils import phase_and_task +from avalanche.evaluation.metric_utils import phase_and_task from avalanche.logging import StrategyLogger From d80613375235d1296929591d1e9cb8af2bcc2e3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xa9aX=20=E3=83=84?= Date: Sat, 25 Sep 2021 11:07:49 +0530 Subject: [PATCH 08/13] Fix type error --- examples/wandb_logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/wandb_logger.py b/examples/wandb_logger.py index 50839113d..ab19b3972 100644 --- a/examples/wandb_logger.py +++ b/examples/wandb_logger.py @@ -133,7 +133,7 @@ def main(args): parser.add_argument('--project', type=str, help='Define the name of the WandB project') parser.add_argument('--run', type=str, help='Provide a run name for WandB') - parser.add_argument('--artifacts', type=bool, default=False, + parser.add_argument('--artifacts', default=False, action="store_true", help='Log Model Checkpoints as W&B Artifacts') parser.add_argument('--path', type=str, default=None, From 3d6e7e4bd14b2a615dd617be8628673e1c16fb2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xa9aX=20=E3=83=84?= Date: Sun, 26 Sep 2021 00:56:26 +0530 Subject: [PATCH 09/13] Update checkpoint metric --- avalanche/evaluation/metrics/checkpoint.py | 1 + avalanche/logging/wandb_logger.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/avalanche/evaluation/metrics/checkpoint.py b/avalanche/evaluation/metrics/checkpoint.py index bef49acae..bf0481283 100644 --- a/avalanche/evaluation/metrics/checkpoint.py +++ b/avalanche/evaluation/metrics/checkpoint.py @@ -82,6 +82,7 @@ def _package_result(self, strategy) -> 'MetricResult': def after_eval_exp(self, strategy: 'BaseStrategy') -> 'MetricResult': model_params = copy.deepcopy(list(strategy.model.parameters())) self.update(model_params) + return self._package_result(strategy) def __str__(self): return "WeightCheckpoint" diff --git a/avalanche/logging/wandb_logger.py b/avalanche/logging/wandb_logger.py index 33869fc1a..648c62247 100644 --- a/avalanche/logging/wandb_logger.py +++ b/avalanche/logging/wandb_logger.py @@ -109,6 +109,10 @@ def before_run(self): self.wandb.run._label(repo="Avalanche") def log_single_metric(self, name, value, x_plot): + + print(f"name: {name}") + print(f"value: {type(value)}") + if isinstance(value, AlternativeValues): value = value.best_supported_value(Image, Tensor, TensorImage, Figure, float, int, From d7473930085660244e36b9464c2bd1ca0eb0c122 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xa9aX=20=E3=83=84?= Date: Sun, 26 Sep 2021 00:59:25 +0530 Subject: [PATCH 10/13] Remove redundant print statements --- avalanche/logging/wandb_logger.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/avalanche/logging/wandb_logger.py b/avalanche/logging/wandb_logger.py index 648c62247..a963feafc 100644 --- a/avalanche/logging/wandb_logger.py +++ b/avalanche/logging/wandb_logger.py @@ -110,9 +110,6 @@ def before_run(self): def log_single_metric(self, name, value, x_plot): - print(f"name: {name}") - print(f"value: {type(value)}") - if isinstance(value, AlternativeValues): value = value.best_supported_value(Image, Tensor, TensorImage, Figure, float, int, From 8d9bc6a2e0ec17d847b3fe1356de7c88f905a80c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xa9aX=20=E3=83=84?= Date: Sun, 26 Sep 2021 01:31:08 +0530 Subject: [PATCH 11/13] Update checkpoint naming --- avalanche/logging/wandb_logger.py | 63 ++++++++++++++++--------------- examples/wandb_logger.py | 2 +- 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/avalanche/logging/wandb_logger.py b/avalanche/logging/wandb_logger.py index a963feafc..2eb91ae38 100644 --- a/avalanche/logging/wandb_logger.py +++ b/avalanche/logging/wandb_logger.py @@ -110,31 +110,7 @@ def before_run(self): def log_single_metric(self, name, value, x_plot): - if isinstance(value, AlternativeValues): - value = value.best_supported_value(Image, Tensor, TensorImage, - Figure, float, int, - self.wandb.viz.CustomChart) - - if not isinstance(value, (Image, Tensor, Figure, float, int, - self.wandb.viz.CustomChart)): - # Unsupported type - return - - if isinstance(value, Image): - self.wandb.log({name: self.wandb.Image(value)}) - - elif isinstance(value, Tensor): - value = np.histogram(value.view(-1).numpy()) - self.wandb.log({name: self.wandb.Histogram(np_histogram=value)}) - - elif isinstance(value, (float, int, Figure, - self.wandb.viz.CustomChart)): - self.wandb.log({name: value}) - - elif isinstance(value, TensorImage): - self.wandb.log({name: self.wandb.Image(array(value))}) - - elif name.startswith("WeightCheckpoint"): + if name.startswith("WeightCheckpoint"): if self.log_artifacts: cwd = os.getcwd() ckpt = os.path.join(cwd, self.path) @@ -143,25 +119,52 @@ def log_single_metric(self, name, value, x_plot): except OSError as e: if e.errno != errno.EEXIST: raise - suffix = '.pth' - dir_name = os.path.join(ckpt, name+suffix) - artifact_name = os.path.join('Models', name+suffix) + ckpt_name = "Model_{}.pth".format(phase_and_task(self.strategy)[1]) + dir_name = os.path.join(ckpt, ckpt_name) + artifact_name = os.path.join('Models', ckpt_name) + if isinstance(value, Tensor): torch.save(value, dir_name) - name = os.path.splittext(self.checkpoint) + model_name = os.path.splittext(self.checkpoint) metadata = {'experience': self.strategy.experience.current_experience, **({'task_id': phase_and_task(self.strategy)[1]} if phase_and_task(self.strategy)[1] else {})} - artifact = self.wandb.Artifact(name, type='model', + artifact = self.wandb.Artifact(model_name, type='model', metadata=metadata) artifact.add_file(dir_name, name=artifact_name) self.wandb.run.log_artifact(artifact) if self.uri is not None: artifact.add_reference(self.uri, name=artifact_name) + else: + if isinstance(value, AlternativeValues): + value = value.best_supported_value(Image, Tensor, TensorImage, + Figure, float, int, + self.wandb.viz.CustomChart) + + if not isinstance(value, (Image, Tensor, Figure, float, int, + self.wandb.viz.CustomChart)): + # Unsupported type + return + + if isinstance(value, Image): + self.wandb.log({name: self.wandb.Image(value)}) + + elif isinstance(value, Tensor): + value = np.histogram(value.view(-1).numpy()) + self.wandb.log({name: self.wandb.Histogram(np_histogram=value)}) + + elif isinstance(value, (float, int, Figure, + self.wandb.viz.CustomChart)): + self.wandb.log({name: value}) + + elif isinstance(value, TensorImage): + self.wandb.log({name: self.wandb.Image(array(value))}) + + __all__ = [ 'WandBLogger' diff --git a/examples/wandb_logger.py b/examples/wandb_logger.py index ab19b3972..ca9ab8d02 100644 --- a/examples/wandb_logger.py +++ b/examples/wandb_logger.py @@ -136,7 +136,7 @@ def main(args): parser.add_argument('--artifacts', default=False, action="store_true", help='Log Model Checkpoints as W&B Artifacts') - parser.add_argument('--path', type=str, default=None, + parser.add_argument('--path', type=str, default="Checkpoint", help='Local path to save the model checkpoints') args = parser.parse_args() main(args) From 201346b3fb8bf1a1c6f597502cf9a6cd682adc00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xa9aX=20=E3=83=84?= Date: Sun, 26 Sep 2021 01:36:43 +0530 Subject: [PATCH 12/13] Update wandb example --- examples/wandb_logger.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/wandb_logger.py b/examples/wandb_logger.py index ca9ab8d02..02cb61974 100644 --- a/examples/wandb_logger.py +++ b/examples/wandb_logger.py @@ -17,6 +17,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from avalanche.evaluation.metrics.checkpoint import WeightCheckpoint from os.path import expanduser @@ -35,7 +36,7 @@ from avalanche.evaluation.metrics import forgetting_metrics, \ accuracy_metrics, loss_metrics, cpu_usage_metrics, \ timing_metrics, gpu_usage_metrics, ram_usage_metrics, disk_usage_metrics, \ - MAC_metrics, confusion_matrix_metrics + MAC_metrics, confusion_matrix_metrics, WeightCheckpoint from avalanche.models import SimpleMLP from avalanche.training.strategies import Naive @@ -103,6 +104,7 @@ def main(args): minibatch=True, epoch=True, experience=True, stream=True), MAC_metrics( minibatch=True, epoch=True, experience=True), + WeightCheckpoint(), loggers=[interactive_logger, wandb_logger] ) From ed916e08d55868e8c4fe17e1ef494517a7bc4381 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xa9aX=20=E3=83=84?= Date: Fri, 1 Oct 2021 06:40:41 +0400 Subject: [PATCH 13/13] Remove interactive logger in W&B example code --- examples/wandb_logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/wandb_logger.py b/examples/wandb_logger.py index 02cb61974..c2c6f77f8 100644 --- a/examples/wandb_logger.py +++ b/examples/wandb_logger.py @@ -105,7 +105,7 @@ def main(args): MAC_metrics( minibatch=True, epoch=True, experience=True), WeightCheckpoint(), - loggers=[interactive_logger, wandb_logger] + loggers=[wandb_logger] ) # CREATE THE STRATEGY INSTANCE (NAIVE)