feat(ppsci): support data_effient_nopt

Author: xiaoyewww

examples/data_efficient_nopt/config/data_efficient_nopt_fno_poisson.yaml

@@ -7,10 +7,11 @@ run_name: r0
 use_ddp: False
 config: pois-64-pretrain-e1_20_m3
 sweep_id: ''
+logdir: exp
 
 train_config:
   default: &DEFAULT
-    num_data_workers: 0
+    num_data_workers: 4
     # model
     model: 'fno'
     depth: 5
@@ -289,7 +290,7 @@ infer_config:
   scales_path: 'data/possion_64/poisson_64_e5_15_train_scale.npy'
   ckpt_path: checkpoint/finetune_b01_m0_n8192.pdparams
 
-  num_data_workers: 0
+  num_data_workers: 4
   subsample: 1
   num_demos: 0
   shuffle: False

examples/data_efficient_nopt/data_efficient_nopt.py

@@ -13,11 +13,11 @@
 # limitations under the License.
 
 import gc
-import logging
 import os
 import random
 from argparse import Namespace
 from collections import OrderedDict
+from os import path as osp
 
 import hydra
 import numpy as np
@@ -44,8 +44,7 @@
 from ppsci.arch.data_efficient_nopt_model import param_norm
 from ppsci.data.dataset.data_efficient_nopt_dataset import MixedDatasetLoader
 from ppsci.data.dataset.data_efficient_nopt_dataset import PoisHelmDatasetLoader
-
-logger = logging.getLogger(__name__)
+from ppsci.utils import logger
 
 
 class Trainer:
@@ -437,7 +436,8 @@ def single_dset_val(self, subset, logs, cutoff=40):
                 del temp_loader
                 break
             count += 1
-            input, label = data
+            input = data[0]
+            label = data[1]
 
             # unsupervised pretrain
             if self.params.mode == "train":
@@ -699,6 +699,7 @@ def inference(config):
     config_name="data_efficient_nopt_fno_poisson",
 )
 def main(config: DictConfig):
+    logger.init_logger("ppsci", osp.join(config.logdir, f"{config.mode}.log"), "info")
     if config.mode == "train" or config.mode == "finetune":
         train(config)
     elif config.mode == "infer":

ppsci/data/dataset/data_efficient_nopt_dataset.py

@@ -15,7 +15,6 @@
 # refs: https://github.com/delta-lab-ai/data_efficient_nopt
 
 import glob
-import logging
 import os
 from typing import Iterator
 from typing import TypeVar
@@ -29,7 +28,7 @@
 from paddle.io import RandomSampler
 from paddle.io import Sampler
 
-logger = logging.getLogger(__name__)
+from ppsci.utils import logger
 
 __all__ = [
     "MultisetSampler",
@@ -556,12 +555,12 @@ def __init__(self, params, location, transform, train):
         if self.train:
             if hasattr(self.params, "train_rand_idx_path"):
                 self.train_rand_idx = np.load(self.params.train_rand_idx_path)
-                logging.info("Randomizing train dataset using given random index path")
+                logger.info("Randomizing train dataset using given random index path")
             else:
                 self.train_rand_idx = range(self.data.shape[0])
             self.train_rand_idx = self.train_rand_idx[self.pt_idxs[0] : self.pt_idxs[1]]
             self.data = self.data[()][self.train_rand_idx, ...]
-            logging.info(
+            logger.info(
                 "Getting only data idx for training set for length: {}".format(
                     len(self.train_rand_idx)
                 )
@@ -576,7 +575,7 @@ def __init__(self, params, location, transform, train):
     def _get_files_stats(self):
         self.file = self.location
         with h5py.File(self.file, "r") as _f:
-            logging.info("Getting file stats from {}".format(self.file))
+            logger.info("Getting file stats from {}".format(self.file))
             if len(_f["fields"].shape) == 4:
                 self.n_demos = None
                 self.n_samples = _f["fields"].shape[0]
@@ -599,7 +598,7 @@ def _get_files_stats(self):
                 self.pt_split = self.params.pt_split
             else:
                 self.pt_split = [0.9, 0.1]
-            logging.info(
+            logger.info(
                 "Split training set into {} for pretrain, {} for train. ".format(
                     self.pt_split[0], self.pt_split[1]
                 )
@@ -619,7 +618,7 @@ def _get_files_stats(self):
             )
         self.n_samples /= self.subsample
         self.n_samples = int(self.n_samples)
-        logging.info(
+        logger.info(
             "Found data at path {}. Number of examples: {}. Image Shape: {} x {}".format(
                 self.location, self.n_samples, self.img_shape_x, self.img_shape_y
             )
@@ -631,12 +630,12 @@ def _get_files_stats(self):
             measure_x = self.scales[-2] / self.img_shape_x
             measure_y = self.scales[-1] / self.img_shape_y
             self.measure = measure_x * measure_y
-            logging.info(
+            logger.info(
                 "Scales for PDE are (source, tensor, sol, domain): {}".format(
                     self.scales
                 )
             )
-            logging.info(
+            logger.info(
                 "Measure of the set is lx/nx * ly/ny =  {}/{} * {}/{}".format(
                     self.scales[-2], self.img_shape_x, self.scales[-1], self.img_shape_y
                 )